diff --git "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" --- "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" +++ "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" @@ -8436,3 +8436,3725 @@ 2021-07-17 02:11:07,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log 2021-07-17 02:11:09,394 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status 2021-07-17 02:11:09,394 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:11:22,800 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:11:24,527 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:11:24,528 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:11:39,659 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:11:39,674 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:11:52,967 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:11:54,861 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:11:54,861 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:12:09,993 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:12:09,994 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:12:19,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:21,749 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:23,039 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:12:23,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:25,128 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:12:25,129 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:12:25,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:27,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:39,490 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:12:39,490 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:12:39,491 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:12:39,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:12:40,272 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:12:40,272 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:12:41,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:43,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:45,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:47,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:12:53,123 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:12:55,412 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:12:55,412 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:12:59,567 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:12:59,567 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:12:59,568 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:12:59,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:13:01,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:04,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:06,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:08,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:10,541 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:13:10,542 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:13:19,816 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:13:19,816 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:13:19,816 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:13:20,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:13:21,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:22,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:23,197 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:13:24,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:25,715 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:13:25,715 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:13:26,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:28,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:39,915 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:13:39,916 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:13:39,916 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:13:40,783 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:13:40,938 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:13:40,938 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:13:41,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:42,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:44,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:46,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:48,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:13:53,273 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:13:56,138 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:13:56,138 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:13:59,994 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:13:59,994 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:13:59,995 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:14:00,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:14:02,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:04,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:06,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:08,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:11,267 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:14:11,267 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:14:20,117 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:14:20,118 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:14:20,118 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:14:20,801 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:20,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:14:22,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:23,342 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:14:24,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:26,401 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:14:26,401 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:14:26,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:28,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:40,219 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:14:40,220 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:14:40,220 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:14:40,810 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:14:41,651 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:14:41,652 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:14:42,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:44,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:46,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:48,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:14:53,417 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:14:56,790 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:14:56,790 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:15:00,369 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:15:00,369 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:15:00,370 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:15:00,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:15:02,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:05,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:07,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:09,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:11,923 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:15:11,923 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:15:20,455 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:15:20,456 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:15:20,456 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:15:20,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:15:21,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:22,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:23,490 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:15:23,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:25,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:27,125 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:15:27,125 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:15:27,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:29,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:40,585 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:15:40,585 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:15:40,586 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:15:40,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:15:41,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:42,280 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:15:42,281 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:15:42,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:43,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:45,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:47,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:49,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:15:53,563 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:15:57,493 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:15:57,494 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:16:00,772 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:16:00,772 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:16:00,773 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:16:00,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:16:01,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:02,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:03,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:05,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:07,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:09,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:12,630 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:16:12,630 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:16:21,009 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:16:21,009 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:16:21,009 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:16:21,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:21,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:16:22,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:23,643 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:16:23,854 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:25,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:27,767 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:16:27,768 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:16:27,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:29,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:41,188 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:16:41,189 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:16:41,189 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:16:41,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:41,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:16:42,966 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:16:42,967 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:16:43,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:45,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:48,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:50,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:16:53,720 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:16:58,143 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:16:58,143 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:17:01,301 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:17:01,302 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:17:01,302 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:17:01,869 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:17:02,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:03,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:04,871 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:06,872 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:08,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:10,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:13,275 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:17:13,276 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:17:21,402 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:17:21,402 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:17:21,403 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:17:21,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:17:22,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:23,796 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:17:23,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:24,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:26,880 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:28,451 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:17:28,452 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:17:28,881 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:30,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:41,484 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:17:41,485 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:17:41,485 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:17:41,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:17:42,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:43,707 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:17:43,707 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:17:43,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:44,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:46,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:48,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:50,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:17:53,865 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:17:58,857 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:17:58,857 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:18:01,569 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:18:01,570 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:18:01,570 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:18:01,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:18:02,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:03,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:04,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:07,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:09,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:13,990 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:18:13,991 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:18:21,652 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:18:21,653 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:18:21,653 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:18:21,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:18:23,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:23,935 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:18:25,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:27,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:29,120 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:18:29,120 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:18:29,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:31,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:41,827 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:18:41,827 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:18:41,828 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:18:41,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:18:43,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:44,417 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:18:44,417 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:18:45,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:47,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:49,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:51,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:18:54,005 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:18:59,604 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:18:59,605 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:19:01,948 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:19:01,948 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:19:01,949 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:19:02,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:19:03,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:05,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:07,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:09,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:11,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:14,737 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:19:14,738 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:19:22,045 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:19:22,045 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:19:22,045 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:19:22,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:19:23,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:24,076 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:19:25,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:27,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:29,867 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:19:29,868 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:19:29,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:31,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:42,204 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:19:42,204 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:19:42,205 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:19:42,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:19:43,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:45,058 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:19:45,058 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:19:45,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:47,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:49,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:51,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:19:54,146 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:20:00,242 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:20:00,242 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:20:02,347 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:20:02,348 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:20:02,348 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:20:02,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:20:03,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:05,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:07,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:09,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:11,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:15,374 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:20:15,374 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:20:22,443 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:20:22,444 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:20:22,444 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:20:22,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:20:23,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:24,213 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:20:24,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:26,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:28,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:30,508 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:20:30,508 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:20:30,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:32,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:42,631 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:20:42,631 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:20:42,632 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:20:42,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:20:44,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:45,697 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:20:45,698 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:20:46,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:48,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:50,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:52,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:20:54,278 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:21:00,888 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:21:00,888 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:21:02,752 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:21:02,752 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:21:02,752 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:21:02,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:21:04,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:06,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:08,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:10,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:12,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:16,020 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:21:16,020 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:21:22,887 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:21:22,887 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:21:22,888 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:21:22,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:21:24,348 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:21:24,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:26,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:28,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:30,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:31,153 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:21:31,153 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:21:42,985 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:21:42,985 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:21:42,985 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:21:42,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:21:44,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:46,343 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:21:46,344 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:21:46,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:48,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:50,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:52,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:21:54,418 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:22:01,590 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:22:01,590 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:22:03,136 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:22:03,136 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:22:03,137 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:22:03,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:22:05,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:07,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:09,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:11,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:16,720 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:22:16,720 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:22:23,301 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:22:23,301 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:22:23,302 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:22:24,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:22:24,491 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:22:25,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:27,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:30,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:31,850 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:22:31,850 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:22:32,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:43,373 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:22:43,373 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:22:43,374 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:22:44,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:22:45,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:46,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:46,985 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:22:46,986 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:22:48,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:50,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:52,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:22:54,562 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:23:02,119 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:23:02,119 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:23:03,535 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:23:03,536 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:23:03,536 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:23:04,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:23:05,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:06,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:08,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:10,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:12,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:17,249 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:23:17,250 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:23:23,688 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:23:23,688 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:23:23,688 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:23:24,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:23:24,637 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:23:25,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:26,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:28,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:30,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:32,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:32,380 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:23:32,380 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:23:34,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:43,834 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:23:43,835 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:23:43,835 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:23:44,044 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:23:45,044 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:46,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:47,519 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:23:47,519 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:23:48,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:50,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:52,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:54,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:23:54,713 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:24:02,651 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:24:02,652 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:24:03,990 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:24:03,990 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:24:03,991 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:24:04,053 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:24:06,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:08,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:10,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:12,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:17,783 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:24:17,784 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:24:24,058 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:24:24,059 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:24:24,059 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:24:24,061 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:24:24,789 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:24:26,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:28,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:30,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:32,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:32,917 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:24:32,917 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:24:44,207 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:24:44,207 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:24:44,208 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:24:45,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:24:46,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:48,061 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:24:48,061 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:24:49,071 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:51,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:53,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:24:54,866 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:25:03,193 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:25:03,194 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:25:04,345 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:25:04,345 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:25:04,346 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:25:05,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:25:06,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:07,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:09,080 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:11,081 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:13,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:18,324 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:25:18,325 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:25:24,486 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:25:24,487 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:25:24,487 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:25:24,939 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:25:25,087 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:25:26,088 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:27,088 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:29,089 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:31,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:33,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:33,454 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:25:33,454 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:25:44,569 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:25:44,569 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:25:44,570 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:25:45,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:25:46,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:47,097 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:48,589 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:25:48,590 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:25:49,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:51,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:53,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:25:55,017 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:26:03,779 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:26:03,780 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:26:04,625 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:26:04,625 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:26:04,625 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:26:05,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:26:06,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:07,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:09,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:11,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:13,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:18,909 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:26:18,910 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:26:24,695 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:26:24,695 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:26:24,696 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:26:25,090 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:26:25,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:26:26,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:27,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:29,115 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:31,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:33,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:34,042 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:26:34,042 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:26:44,802 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:26:44,803 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:26:44,803 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:26:45,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:26:47,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:49,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:49,233 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:26:49,234 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:26:51,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:53,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:26:55,166 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:27:04,375 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:27:04,375 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:27:04,898 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:27:04,898 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:27:04,899 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:27:05,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:27:07,131 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:10,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:12,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:14,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:19,507 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:27:19,507 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:27:25,049 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:27:25,049 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:27:25,049 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:27:25,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:27:25,238 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:27:26,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:27,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:28,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:30,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:32,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:34,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:34,639 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:27:34,639 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:27:45,156 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:27:45,156 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:27:45,156 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:27:46,148 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:46,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:27:47,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:48,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:49,807 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:27:49,808 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:27:50,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:52,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:54,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:27:55,313 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:28:04,941 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:28:04,941 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:28:05,235 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:28:05,235 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:28:05,236 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:28:06,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:06,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:28:07,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:08,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:10,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:13,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:15,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:20,086 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:28:20,086 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:28:25,283 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:28:25,283 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:28:25,283 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:28:25,390 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:28:26,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:28:27,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:29,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:31,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:33,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:35,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:35,221 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:28:35,221 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:28:45,443 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:28:45,444 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:28:45,444 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:28:46,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:28:47,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:49,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:50,405 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:28:50,406 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:28:51,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:53,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:55,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:28:55,468 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:29:05,586 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:29:05,587 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:29:05,715 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:29:05,716 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:29:05,716 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:29:06,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:29:07,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:09,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:11,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:13,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:15,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:20,723 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:29:20,723 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:29:25,545 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:29:25,720 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:29:25,721 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:29:25,721 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:29:26,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:29:27,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:29,193 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:31,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:33,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:35,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:35,856 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:29:35,857 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:29:45,928 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:29:45,928 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:29:45,929 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:29:46,201 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:29:47,201 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:49,202 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:51,074 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:29:51,074 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:29:51,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:53,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:55,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:29:55,622 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:30:06,046 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:30:06,047 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:30:06,047 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:30:06,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:30:06,332 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:30:06,333 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:30:07,210 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:09,211 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:12,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:14,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:21,527 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:30:21,527 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:30:25,696 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:30:26,172 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:30:26,172 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:30:26,173 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:30:26,218 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:30:28,219 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:30,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:32,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:34,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:36,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:36,658 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:30:36,658 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:30:46,245 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:30:46,246 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:30:46,246 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:30:47,228 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:30:48,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:50,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:51,870 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:30:51,870 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:30:52,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:54,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:30:55,772 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:30:56,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:06,318 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:31:06,318 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:31:06,319 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:31:07,093 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:31:07,093 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:31:07,237 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:31:08,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:10,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:12,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:14,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:22,243 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:31:22,244 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:31:25,844 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:31:26,379 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:31:26,379 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:31:26,380 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:31:27,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:31:28,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:30,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:32,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:34,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:37,375 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:31:37,375 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:31:46,479 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:31:46,480 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:31:46,480 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:31:47,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:31:48,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:50,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:52,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:52,505 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:31:52,506 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:31:54,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:31:55,918 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:32:06,569 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:32:06,570 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:32:06,570 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:32:07,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:32:07,673 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:32:07,673 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:32:08,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:10,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:12,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:14,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:22,809 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:32:22,809 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:32:25,992 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:32:26,646 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:32:26,647 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:32:26,647 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:32:27,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:32:28,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:29,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:31,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:33,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:35,279 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:37,941 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:32:37,942 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:32:46,729 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:32:46,729 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:32:46,730 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:32:47,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:32:48,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:49,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:51,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:53,112 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:32:53,112 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:32:53,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:55,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:32:56,065 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:33:06,808 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:33:06,808 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:33:06,809 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:33:07,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:33:08,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:08,354 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:33:08,355 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:33:09,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:11,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:13,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:15,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:23,552 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:33:23,553 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:33:26,138 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:33:26,914 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:33:26,915 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:33:26,915 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:33:27,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:33:28,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:29,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:31,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:33,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:35,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:38,686 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:33:38,687 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:33:47,004 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:33:47,004 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:33:47,005 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:33:47,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:33:48,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:49,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:51,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:53,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:53,839 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:33:53,839 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:33:55,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:33:56,213 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:34:07,175 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:34:07,175 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:34:07,176 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:34:07,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:34:08,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:08,994 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:34:08,995 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:34:09,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:11,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:13,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:15,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:24,180 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:34:24,180 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:34:26,285 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:34:27,236 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:34:27,236 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:34:27,237 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:34:27,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:34:29,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:31,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:33,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:35,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:39,311 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:34:39,311 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:34:47,304 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:34:47,304 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:34:47,305 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:34:47,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:34:49,338 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:51,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:53,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:54,445 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:34:54,446 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:34:55,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:34:56,361 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:35:07,429 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:35:07,429 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:35:07,429 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:35:08,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:35:09,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:09,733 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:35:09,733 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:35:10,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:12,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:14,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:16,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:24,869 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:35:24,870 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:35:26,437 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:35:27,521 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:35:27,521 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:35:27,522 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:35:28,356 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:35:29,356 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:30,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:32,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:34,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:36,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:40,002 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:35:40,002 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:35:47,574 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:35:47,574 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:35:47,575 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:35:48,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:35:49,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:50,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:52,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:54,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:55,135 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:35:55,136 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:35:56,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:35:56,510 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:36:07,665 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:36:07,666 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:36:07,666 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:36:08,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:36:09,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:10,339 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:36:10,339 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:36:10,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:12,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:14,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:16,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:25,532 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:36:25,532 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:36:26,584 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:36:27,800 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:36:27,800 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:36:27,801 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:36:28,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:36:29,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:30,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:32,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:34,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:36,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:40,681 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:36:40,681 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:36:47,933 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:36:47,934 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:36:47,934 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:36:48,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:36:49,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:50,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:52,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:54,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:55,812 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:36:55,812 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:36:56,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:36:56,657 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:37:08,085 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:37:08,085 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:37:08,086 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:37:08,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:37:09,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:10,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:10,951 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:37:10,952 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:37:12,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:14,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:16,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:26,117 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:37:26,117 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:37:26,725 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:37:28,164 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:37:28,164 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:37:28,165 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:37:28,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:37:29,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:30,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:32,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:34,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:36,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:41,249 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:37:41,249 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:37:48,320 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:37:48,321 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:37:48,321 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:37:48,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:37:50,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:51,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:53,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:55,423 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:37:56,379 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:37:56,380 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:37:56,793 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:37:57,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:08,376 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:38:08,377 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:38:08,377 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:38:08,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:38:09,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:10,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:11,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:11,653 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:38:11,653 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:38:13,431 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:15,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:17,433 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:26,804 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:38:26,805 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:38:26,940 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:38:28,430 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:38:28,431 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:38:28,431 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:38:28,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:38:29,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:30,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:31,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:33,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:35,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:37,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:41,942 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:38:41,942 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:38:48,572 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:38:48,573 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:38:48,573 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:38:49,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:49,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:38:50,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:51,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:53,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:55,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:38:56,930 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:38:57,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:38:57,076 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:38:57,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:08,700 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:39:08,700 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:39:08,701 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:39:09,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:09,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:39:10,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:11,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:12,232 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:39:12,233 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:39:13,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:16,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:18,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:26,997 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:39:27,414 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:39:27,414 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:39:28,785 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:39:28,786 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:39:28,788 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:39:29,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:39:30,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:32,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:34,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:36,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:38,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:42,545 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:39:42,545 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:39:48,914 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:39:48,914 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:39:48,915 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:39:49,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:39:50,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:52,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:54,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:56,477 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:39:57,067 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:39:57,679 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:39:57,679 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:39:58,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:09,114 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:40:09,115 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:40:09,117 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:40:09,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:40:10,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:12,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:13,047 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:40:13,047 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:40:14,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:16,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:18,486 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:27,135 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:40:28,190 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:40:28,190 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:40:29,196 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:40:29,197 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:40:29,197 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:40:29,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:40:30,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:32,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:34,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:36,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:38,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:43,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:40:43,322 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:40:49,357 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:40:49,357 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:40:49,359 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:40:49,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:40:50,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:51,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:52,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:54,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:56,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:40:57,208 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:40:58,455 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:40:58,456 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:40:58,504 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:09,445 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:41:09,445 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:41:09,446 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:41:09,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:41:10,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:11,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:12,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:13,624 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:41:13,624 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:41:14,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:16,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:18,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:27,282 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:41:28,812 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:41:28,813 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:41:29,546 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:41:29,546 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:41:29,547 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:41:30,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:30,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:41:31,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:32,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:34,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:36,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:39,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:43,945 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:41:43,946 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:41:49,602 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:41:49,602 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:41:49,602 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:41:50,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:41:51,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:53,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:55,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:57,355 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:41:57,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:41:59,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:41:59,076 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:41:59,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:09,710 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:42:09,711 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:42:09,711 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:42:10,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:42:11,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:13,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:14,244 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:42:14,244 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:42:15,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:17,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:19,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:27,427 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:42:29,441 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:42:29,441 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:42:29,830 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:42:29,830 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:42:29,831 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:42:30,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:42:31,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:33,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:35,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:37,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:44,573 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:42:44,573 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:42:49,881 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:42:49,881 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:42:49,882 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:42:50,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:42:51,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:53,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:55,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:57,497 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:42:57,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:59,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:42:59,704 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:42:59,704 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:43:09,967 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:43:09,967 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:43:09,968 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:43:10,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:43:11,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:13,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:14,906 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:43:14,906 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:43:15,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:17,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:27,569 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:43:30,079 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:43:30,079 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:43:30,079 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:43:30,168 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:43:30,168 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:43:30,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:43:31,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:32,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:34,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:36,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:38,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:45,332 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:43:45,332 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:43:50,241 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:43:50,241 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:43:50,242 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:43:50,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:43:51,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:52,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:54,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:56,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:43:57,644 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:43:58,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:00,461 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:44:00,461 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:44:10,454 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:44:10,454 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:44:10,454 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:44:10,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:44:11,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:12,586 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:14,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:15,700 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:44:15,701 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:44:16,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:18,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:27,716 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:44:30,635 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:44:30,635 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:44:30,636 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:44:30,963 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:44:30,964 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:44:31,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:44:32,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:34,595 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:36,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:38,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:46,114 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:44:46,114 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:44:50,746 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:44:50,747 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:44:50,747 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:44:51,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:44:52,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:54,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:56,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:44:57,790 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:44:58,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:01,248 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:45:01,248 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:45:10,879 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:45:10,879 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:45:10,880 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:45:11,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:45:12,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:14,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:16,429 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:45:16,429 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:45:16,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:18,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:27,858 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:45:31,012 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:45:31,012 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:45:31,013 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:45:31,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:45:31,656 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:45:31,657 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:45:32,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:34,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:36,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:38,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:46,848 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:45:46,849 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:45:51,098 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:45:51,099 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:45:51,099 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:45:51,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:45:52,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:54,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:56,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:45:57,932 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:45:58,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:01,981 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:46:01,982 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:46:11,179 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:46:11,180 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:46:11,180 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:46:11,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:46:12,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:14,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:16,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:17,232 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:46:17,232 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:46:18,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:20,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:28,006 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:46:31,251 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:46:31,252 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:46:31,252 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:46:31,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:46:32,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:46:32,461 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:46:32,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:35,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:37,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:39,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:47,625 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:46:47,625 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:46:51,395 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:46:51,395 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:46:51,396 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:46:51,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:46:52,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:53,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:55,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:57,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:46:58,079 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:46:59,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:02,758 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:47:02,758 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:47:11,555 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:47:11,555 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:47:11,555 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:47:11,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:47:12,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:13,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:15,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:17,671 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:17,958 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:47:17,958 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:47:19,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:28,153 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:47:31,655 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:47:31,656 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:47:31,656 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:47:31,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:47:32,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:33,211 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:47:33,212 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:47:33,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:35,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:37,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:39,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:48,356 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:47:48,356 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:47:51,807 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:47:51,808 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:47:51,808 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:47:52,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:47:53,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:55,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:57,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:47:58,229 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:47:59,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:03,488 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:48:03,488 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:48:11,932 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:48:11,933 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:48:11,933 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:48:12,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:48:13,697 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:15,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:17,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:18,616 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:48:18,616 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:48:19,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:28,299 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:48:32,035 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:48:32,036 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:48:32,036 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:48:32,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:48:33,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:33,749 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:48:33,749 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:48:35,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:37,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:39,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:48,940 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:48:48,940 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:48:52,125 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:48:52,125 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:48:52,126 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:48:52,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:48:53,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:56,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:48:58,372 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:48:58,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:00,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:04,081 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:49:04,082 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:49:12,190 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:49:12,190 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:49:12,191 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:49:12,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:49:13,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:14,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:16,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:18,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:19,217 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:49:19,217 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:49:20,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:28,446 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:49:32,331 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:49:32,331 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:49:32,332 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:49:32,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:49:33,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:34,381 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:49:34,381 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:49:34,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:36,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:38,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:40,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:49,579 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:49:49,579 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:49:52,526 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:49:52,526 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:49:52,526 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:49:52,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:49:53,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:54,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:56,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:49:58,524 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:49:58,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:00,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:04,719 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:50:04,719 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:50:12,641 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:50:12,642 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:50:12,642 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:50:12,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:50:13,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:14,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:16,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:18,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:19,849 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:50:19,849 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:50:20,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:28,600 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:50:32,756 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:50:32,756 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:50:32,757 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:50:32,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:50:33,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:34,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:34,981 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:50:34,981 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:50:36,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:39,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:41,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:50,114 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:50:50,114 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:50:52,836 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:50:52,837 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:50:52,837 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:50:53,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:53,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:50:55,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:57,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:50:58,676 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:50:59,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:01,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:05,244 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:51:05,245 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:51:12,920 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:51:12,921 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:51:12,921 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:51:13,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:51:14,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:15,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:17,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:19,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:20,375 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:51:20,376 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:51:21,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:28,752 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:51:33,002 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:51:33,003 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:51:33,003 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:51:33,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:51:34,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:35,538 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:51:35,538 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:51:35,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:37,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:39,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:41,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:51,765 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:51:51,766 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:51:53,108 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:51:53,109 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:51:53,109 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:51:53,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:51:54,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:55,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:57,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:51:58,829 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:51:59,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:01,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:06,914 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:52:06,915 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:52:13,235 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:52:13,236 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:52:13,236 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:52:13,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:52:14,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:15,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:17,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:19,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:21,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:22,045 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:52:22,045 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:52:28,904 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:52:33,392 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:52:33,393 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:52:33,393 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:52:33,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:33,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:52:34,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:35,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:37,183 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:52:37,183 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:52:37,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:39,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:41,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:52,387 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:52:52,387 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:52:53,622 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:52:53,622 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:52:53,622 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:52:53,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:52:54,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:56,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:58,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:52:58,979 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:53:00,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:02,826 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:07,520 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:53:07,520 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:53:13,724 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:53:13,725 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:53:13,725 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:53:13,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:53:14,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:16,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:18,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:20,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:22,652 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:53:22,653 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:53:22,835 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:29,057 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:53:33,830 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:53:33,831 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:53:33,831 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:53:33,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:53:34,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:36,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:37,815 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:53:37,815 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:53:38,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:40,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:42,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:52,960 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:53:52,961 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:53:53,881 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:53:53,881 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:53:53,882 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:53:54,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:54,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:53:55,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:56,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:58,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:53:59,135 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:54:00,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:02,854 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:08,095 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:54:08,096 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:54:13,954 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:54:13,954 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:54:13,955 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:54:14,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:14,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:54:15,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:16,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:18,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:20,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:22,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:23,226 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:54:23,227 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:54:29,211 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:54:34,090 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:54:34,091 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:54:34,091 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:54:34,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:34,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:54:35,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:36,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:38,401 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:54:38,401 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:54:38,869 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:40,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:42,871 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:53,578 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:54:53,579 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:54:54,205 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:54:54,206 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:54:54,206 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:54:54,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:54,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:54:55,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:56,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:54:59,286 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:54:59,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:01,880 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:03,881 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:08,709 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:55:08,709 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:55:14,280 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:55:14,280 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:55:14,281 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:55:14,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:55:15,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:17,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:19,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:21,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:23,839 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:55:23,839 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:55:23,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:29,363 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:55:34,411 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:55:34,411 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:55:34,412 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:55:34,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:55:35,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:37,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:39,040 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:55:39,040 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:55:39,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:41,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:43,899 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:54,205 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:55:54,205 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:55:54,511 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:55:54,511 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:55:54,512 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:55:54,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:55:55,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:57,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:55:59,437 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:55:59,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:01,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:09,340 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:56:09,341 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:56:14,563 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:56:14,563 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:56:14,564 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:56:14,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:56:15,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:17,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:19,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:21,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:23,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:24,476 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:56:24,477 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:56:29,505 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:56:34,599 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:56:34,599 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:56:34,600 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:56:34,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:56:35,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:37,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:39,662 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:56:39,663 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:56:39,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:41,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:43,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:54,673 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:56:54,673 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:56:54,674 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:56:54,897 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:56:54,898 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:56:54,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:56:55,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:56,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:57,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:56:59,587 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:56:59,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:01,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:10,051 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:57:10,051 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:57:14,797 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:57:14,797 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:57:14,798 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:57:14,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:57:16,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:18,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:20,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:22,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:25,182 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:57:25,182 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:57:29,658 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:57:34,874 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:57:34,874 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:57:34,875 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:57:34,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:57:36,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:38,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:40,334 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:57:40,334 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:57:40,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:42,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:44,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:54,965 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:57:54,965 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:57:54,966 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:57:55,543 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:57:55,544 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:57:55,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:57:56,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:58,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:57:59,733 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:58:00,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:02,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:10,703 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:58:10,704 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:58:15,091 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:58:15,091 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:58:15,091 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:58:15,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:58:16,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:18,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:20,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:22,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:25,834 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:58:25,834 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:58:29,803 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:58:35,286 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:58:35,286 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:58:35,287 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:58:35,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:58:36,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:38,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:40,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:41,012 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:58:41,012 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:58:42,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:44,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:55,382 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:58:55,382 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:58:55,383 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:58:55,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:58:56,243 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:58:56,243 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:58:56,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:58,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:58:59,867 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:59:00,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:02,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:11,408 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:59:11,409 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:59:15,450 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:59:15,450 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:59:15,451 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:59:15,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:59:16,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:18,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:22,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:24,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:26,544 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:59:26,544 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:59:29,935 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:59:35,581 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:59:35,581 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:59:35,582 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:59:36,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:59:38,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:40,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:41,675 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:59:41,676 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:59:42,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:44,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:59:55,681 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:59:55,682 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:59:55,682 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:59:56,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:59:56,877 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:59:56,877 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:59:58,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:00,003 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:00:00,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:02,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:04,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:12,010 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:00:12,010 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:00:15,786 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:00:15,787 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:00:15,787 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:00:16,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:00:18,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:20,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:22,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:24,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:27,144 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:00:27,144 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:00:30,073 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:00:35,876 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:00:35,876 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:00:35,876 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:00:36,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:00:38,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:40,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:42,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:42,324 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:00:42,324 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:00:44,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:00:55,950 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:00:55,950 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:00:55,951 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:00:56,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:00:57,547 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:00:57,548 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:00:58,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:00,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:00,145 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:01:02,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:04,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:12,722 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:01:12,723 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:01:16,071 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:01:16,071 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:01:16,072 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:01:17,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:01:18,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:20,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:22,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:24,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:27,856 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:01:27,856 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:01:30,213 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:01:36,219 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:01:36,219 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:01:36,220 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:01:37,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:01:38,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:40,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:42,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:42,987 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:01:42,987 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:01:44,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:56,272 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:01:56,272 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:01:56,273 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:01:57,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:01:58,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:01:58,277 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:01:58,277 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:02:00,288 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:02:01,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:03,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:05,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:13,446 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:02:13,447 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:02:16,457 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:02:16,458 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:02:16,458 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:02:17,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:02:18,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:19,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:21,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:23,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:25,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:28,578 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:02:28,579 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:02:30,360 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:02:36,569 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:02:36,569 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:02:36,570 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:02:37,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:02:38,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:39,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:41,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:43,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:43,711 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:02:43,711 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:02:45,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:56,678 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:02:56,678 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:02:56,678 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:02:57,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:02:58,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:02:58,841 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:02:58,842 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:02:59,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:00,430 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:03:01,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:03,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:05,104 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:13,976 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:03:13,977 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:03:16,747 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:03:16,747 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:03:16,748 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:03:17,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:03:19,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:21,111 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:23,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:25,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:29,108 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:03:29,109 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:03:30,500 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:03:36,839 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:03:36,839 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:03:36,840 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:03:37,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:03:39,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:41,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:43,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:44,245 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:03:44,245 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:03:45,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:56,981 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:03:56,981 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:03:56,981 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:03:57,129 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:03:59,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:03:59,441 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:03:59,442 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:04:00,572 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:04:01,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:03,131 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:05,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:14,598 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:04:14,599 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:04:17,066 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:04:17,066 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:04:17,066 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:04:17,138 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:04:19,139 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:21,139 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:23,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:25,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:29,730 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:04:29,730 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:04:30,645 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:04:37,187 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:04:37,188 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:04:37,188 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:04:38,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:04:39,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:41,148 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:43,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:44,860 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:04:44,861 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:04:45,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:04:57,255 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:04:57,255 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:04:57,256 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:04:58,156 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:04:59,156 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:00,019 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:05:00,020 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:05:00,721 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:05:01,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:03,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:05,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:15,177 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:05:15,178 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:05:17,360 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:05:17,360 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:05:17,360 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:05:18,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:05:19,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:20,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:22,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:24,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:26,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:30,309 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:05:30,310 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:05:30,795 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:05:37,678 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:05:37,679 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:05:37,679 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:05:38,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:05:39,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:40,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:42,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:44,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:45,445 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:05:45,446 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:05:46,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:05:57,817 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:05:57,818 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:05:57,818 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:05:58,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:05:59,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:00,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:00,609 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:06:00,610 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:06:00,871 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:06:02,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:04,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:06,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:15,796 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:06:15,797 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:06:17,904 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:06:17,905 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:06:17,905 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:06:18,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:06:19,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:20,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:22,193 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:24,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:26,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:30,927 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:06:30,927 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:06:31,058 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:06:38,028 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:06:38,029 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:06:38,029 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:06:38,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:06:40,201 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:42,202 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:44,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:46,059 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:06:46,059 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:06:46,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:06:58,100 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:06:58,101 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:06:58,101 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:06:58,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:07:00,210 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:01,020 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:07:01,222 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:07:01,223 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:07:02,211 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:04,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:06,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:16,361 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:07:16,361 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:07:18,163 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:07:18,164 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:07:18,164 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:07:18,219 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:07:20,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:22,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:24,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:26,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:31,095 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:07:31,492 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:07:31,492 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:07:38,271 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:07:38,272 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:07:38,274 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:07:39,228 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:07:40,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:42,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:45,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:46,623 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:07:46,623 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:07:47,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:07:58,384 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:07:58,384 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:07:58,385 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:07:59,237 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:08:00,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:01,171 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:08:01,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:01,833 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:08:01,834 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:08:03,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:05,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:07,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:16,983 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:08:16,983 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:08:18,509 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:08:18,510 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:08:18,512 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:08:19,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:08:20,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:21,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:23,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:25,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:27,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:31,247 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:08:32,114 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:08:32,114 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:08:38,570 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:08:38,571 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:08:38,571 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:08:39,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:08:40,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:41,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:43,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:45,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:47,246 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:08:47,246 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:08:47,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:58,647 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:08:58,648 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:08:58,651 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:08:59,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:08:59,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:09:00,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:01,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:01,323 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:09:02,430 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:09:02,430 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:09:03,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:05,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:07,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:17,561 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:09:17,562 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:09:18,739 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:09:18,739 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:09:18,740 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:09:19,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:19,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:09:20,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:21,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:23,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:25,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:27,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:31,396 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:09:32,694 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:09:32,694 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:09:38,786 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:09:38,787 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:09:38,788 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:09:39,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:09:41,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:43,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:45,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:47,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:09:47,824 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:09:47,824 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:09:58,904 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:09:58,905 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:09:58,905 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:09:59,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:10:01,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:01,470 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:10:02,973 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:10:02,974 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:10:03,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:06,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:08,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:18,142 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:10:18,142 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:10:18,999 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:10:18,999 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:10:19,000 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:10:19,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:10:20,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:21,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:22,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:24,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:26,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:28,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:31,546 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:10:33,273 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:10:33,273 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:10:39,111 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:10:39,112 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:10:39,112 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:10:39,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:10:40,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:41,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:42,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:44,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:46,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:48,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:10:48,406 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:10:48,406 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:10:59,196 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:10:59,196 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:10:59,197 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:10:59,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:11:00,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:01,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:01,620 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:11:02,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:03,562 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:11:03,563 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:11:04,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:06,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:08,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:18,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:11:18,695 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:11:19,326 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:11:19,327 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:11:19,327 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:11:19,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:11:20,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:21,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:22,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:24,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:26,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:28,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:31,696 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:11:33,823 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:11:33,824 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:11:39,471 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:11:39,471 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:11:39,472 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:11:40,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:40,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:11:41,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:42,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:44,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:46,345 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:48,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:11:48,956 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:11:48,956 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:11:59,597 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:11:59,598 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:11:59,598 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:12:00,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:00,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:12:01,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:01,772 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:12:02,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:04,089 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:12:04,090 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:12:04,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:06,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:08,355 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:19,229 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:12:19,229 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:12:19,661 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:12:19,662 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:12:19,662 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:12:20,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:20,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:12:21,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:22,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:24,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:26,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:28,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:31,851 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:12:34,362 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:12:34,362 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:12:39,798 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:12:39,799 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:12:39,799 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:12:40,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:40,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:12:41,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:42,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:44,372 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:46,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:49,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:12:49,494 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:12:49,494 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:12:59,914 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:12:59,915 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:12:59,915 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:13:00,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:13:01,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:01,928 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:13:03,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:04,683 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:13:04,683 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:13:05,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:07,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:09,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:19,843 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:13:19,843 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:13:19,973 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:13:19,973 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:13:19,974 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:13:20,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:13:21,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:22,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:23,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:25,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:27,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:29,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:32,007 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:13:35,040 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:13:35,041 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:13:40,108 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:13:40,109 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:13:40,109 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:13:40,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:13:41,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:42,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:43,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:45,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:47,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:49,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:13:50,174 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:13:50,174 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:14:00,199 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:14:00,200 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:14:00,200 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:14:00,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:14:01,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:02,084 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:14:02,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:03,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:05,312 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:14:05,313 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:14:05,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:07,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:09,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:20,337 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:14:20,337 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:14:20,337 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:14:20,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:14:20,566 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:14:20,567 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:14:21,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:22,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:23,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:25,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:27,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:29,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:32,162 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:14:35,724 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:14:35,725 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:14:40,398 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:14:40,398 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:14:40,399 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:14:40,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:14:41,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:42,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:43,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:45,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:47,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:49,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:14:50,858 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:14:50,858 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:15:00,585 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:15:00,585 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:15:00,586 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:15:01,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:01,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:15:02,238 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:15:02,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:03,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:05,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:05,990 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:15:05,991 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:15:07,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:09,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:20,638 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:15:20,638 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:15:20,638 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:15:21,153 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:15:21,154 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:15:21,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:21,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:15:22,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:23,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:26,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:28,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:30,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:32,315 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:15:36,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:15:36,322 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:15:40,753 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:15:40,754 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:15:40,754 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:15:41,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:15:42,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:44,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:46,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:48,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:50,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:15:51,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:15:51,461 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:16:00,843 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:16:00,844 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:16:00,844 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:16:01,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:16:02,393 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:16:02,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:04,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:06,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:06,622 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:16:06,622 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:16:08,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:10,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:20,916 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:16:20,916 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:16:20,917 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:16:21,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:16:21,783 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:16:21,784 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:16:22,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:24,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:26,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:28,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:30,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:32,466 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:16:36,918 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:16:36,919 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:16:41,002 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:16:41,002 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:16:41,002 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:16:41,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:16:42,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:44,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:46,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:48,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:16:52,049 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:16:52,049 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:17:01,126 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:17:01,126 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:17:01,127 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:17:01,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:17:02,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:02,541 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:17:04,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:07,242 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:17:07,242 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:17:07,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:09,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:21,257 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:17:21,258 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:17:21,258 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:17:21,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:17:22,410 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:17:22,411 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:17:22,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:23,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:25,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:27,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:29,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:32,615 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:17:37,562 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:17:37,563 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:17:41,346 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:17:41,346 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:17:41,347 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:17:41,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:17:43,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:45,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:47,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:49,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:17:52,696 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:17:52,696 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:18:01,491 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:18:01,491 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:18:01,492 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:18:01,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:18:02,686 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:18:03,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:05,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:07,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:07,827 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:18:07,828 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:18:09,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:21,659 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:18:21,659 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:18:21,660 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:18:22,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:18:22,989 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:18:22,990 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:18:23,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:25,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:27,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:29,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:32,757 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:18:38,155 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:18:38,156 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:18:41,848 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:18:41,849 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:18:41,849 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:18:42,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:18:43,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:45,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:47,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:49,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:18:53,289 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:18:53,290 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:19:01,925 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:19:01,926 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:19:01,926 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:19:02,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:19:02,827 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:19:03,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:05,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:07,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:08,422 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:19:08,423 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:19:09,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:22,071 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:19:22,072 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:19:22,072 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:19:22,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:19:23,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:23,599 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:19:23,599 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:19:24,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:26,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:28,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:30,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:32,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:32,892 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:19:34,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:36,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:38,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:38,735 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:19:38,735 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:19:40,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:42,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:44,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:46,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:48,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:50,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:52,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:53,898 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:19:53,898 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:19:54,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:56,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:19:58,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:00,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:02,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:02,955 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:20:04,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:06,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:08,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:09,031 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:20:09,032 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:20:10,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:12,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:14,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:16,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:18,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:20,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:22,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:24,163 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:20:24,163 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:20:25,932 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:20:25,933 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:20:25,933 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:20:26,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:20:28,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:30,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:32,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:33,020 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:20:34,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:39,293 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:20:39,294 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:20:45,993 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:20:45,993 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:20:45,994 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:20:46,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:46,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:20:48,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:50,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:52,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:20:54,423 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:20:54,423 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:20:54,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:03,096 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:21:06,085 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:21:06,085 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:21:06,086 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:21:06,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:21:08,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:09,559 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:21:09,560 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:21:10,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:13,601 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:15,602 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:24,697 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:21:24,698 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:21:26,171 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:21:26,171 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:21:26,172 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:21:26,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:21:27,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:28,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:29,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:31,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:33,171 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:21:33,610 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:35,611 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:39,830 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:21:39,830 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:21:46,286 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:21:46,287 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:21:46,287 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:21:46,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:21:47,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:48,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:49,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:51,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:53,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:21:54,960 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:21:54,961 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:21:55,620 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:03,246 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:22:06,341 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:22:06,342 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:22:06,342 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:22:06,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:22:07,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:08,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:09,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:10,153 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:22:10,153 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:22:11,628 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:13,629 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:15,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:25,301 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:22:25,302 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:22:26,500 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:22:26,501 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:22:26,501 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:22:26,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:22:27,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:28,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:29,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:31,638 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:33,320 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:22:33,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:35,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:40,433 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:22:40,434 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:22:46,561 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:22:46,562 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:22:46,562 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:22:46,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:22:47,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:48,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:49,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:51,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:53,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:22:55,564 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:22:55,564 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:22:55,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:03,396 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:23:06,668 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:23:06,668 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:23:06,669 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:23:07,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:07,654 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:23:08,654 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:10,655 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:10,765 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:23:10,765 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:23:12,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:14,657 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:25,929 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:23:25,930 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:23:26,773 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:23:26,774 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:23:26,774 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:23:27,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:23:28,664 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:30,664 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:32,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:33,471 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:23:34,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:36,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:41,061 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:23:41,061 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:23:46,839 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:23:46,840 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:23:46,840 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:23:47,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:23:48,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:50,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:52,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:54,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:23:56,195 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:23:56,195 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:23:56,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:03,546 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:24:07,015 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:24:07,016 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:24:07,016 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:24:07,682 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:24:08,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:10,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:11,350 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:24:11,350 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:24:12,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:14,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:16,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:26,545 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:24:26,545 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:24:27,106 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:24:27,107 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:24:27,107 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:24:27,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:24:28,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:30,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:32,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:33,619 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:24:34,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:41,676 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:24:41,676 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:24:47,189 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:24:47,190 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:24:47,191 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:24:47,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:24:48,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:50,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:52,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:54,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:24:56,806 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:24:56,807 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:25:03,694 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:25:07,270 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:25:07,271 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:25:07,271 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:25:07,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:25:08,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:09,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:10,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:11,941 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:25:11,941 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:25:12,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:14,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:27,119 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:25:27,120 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:25:27,420 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:25:27,421 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:25:27,421 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:25:27,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:25:28,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:29,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:30,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:32,721 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:33,770 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:25:34,722 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:42,260 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:25:42,260 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:25:47,489 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:25:47,490 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:25:47,490 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:25:47,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:25:48,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:49,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:51,729 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:53,730 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:55,731 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:25:57,394 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:25:57,394 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:26:03,845 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:26:07,568 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:26:07,569 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:26:07,569 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:26:07,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:26:09,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:11,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:12,551 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:26:12,552 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:26:13,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:15,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:27,691 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:26:27,692 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:26:27,692 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:26:27,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:26:27,794 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:26:27,794 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:26:29,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:31,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:33,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:33,921 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:26:35,749 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:42,927 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:26:42,928 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:26:47,759 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:26:47,760 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:26:47,760 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:26:48,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:26:49,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:51,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:53,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:55,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:26:58,079 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:26:58,079 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:27:03,997 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:27:07,907 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:27:07,908 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:27:07,908 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:27:08,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:27:09,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:11,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:13,213 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:27:13,218 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:27:13,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:15,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:28,054 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:27:28,054 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:27:28,055 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:27:28,382 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:27:28,383 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:27:28,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:27:29,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:31,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:34,077 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:27:34,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:36,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:43,590 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:27:43,590 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:27:48,176 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:27:48,176 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:27:48,177 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:27:48,783 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:27:50,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:52,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:54,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:56,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:27:58,725 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:27:58,725 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:28:04,155 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:28:08,290 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:28:08,290 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:28:08,291 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:28:08,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:28:10,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:12,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:13,887 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:28:13,887 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:28:14,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:16,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:28,400 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:28:28,400 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:28:28,401 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:28:28,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:28:29,050 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:28:29,050 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:28:30,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:32,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:34,224 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:28:34,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:36,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:44,189 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:28:44,190 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:28:48,514 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:28:48,514 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:28:48,514 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:28:48,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:28:50,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:52,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:54,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:56,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:28:59,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:28:59,353 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:29:04,293 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:29:08,622 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:29:08,623 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:29:08,623 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:29:08,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:29:10,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:12,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:14,534 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:29:14,534 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:29:14,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:16,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:28,686 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:29:28,686 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:29:28,687 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:29:28,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:29:29,697 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:29:29,698 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:29:30,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:32,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:34,361 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:29:35,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:37,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:44,840 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:29:44,840 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:29:48,789 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:29:48,790 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:29:48,790 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:29:48,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:29:50,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:51,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:53,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:55,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:57,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:29:59,973 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:29:59,974 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:30:04,430 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:30:08,839 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:30:08,840 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:30:08,840 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:30:08,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:30:09,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:10,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:11,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:13,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:15,128 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:30:15,128 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:30:15,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:17,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:29,003 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:30:29,004 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:30:29,004 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:30:29,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:29,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:30:30,263 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:30:30,263 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:30:30,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:31,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:33,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:34,499 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:30:35,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:37,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:45,491 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:30:45,492 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:30:49,115 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:30:49,115 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:30:49,116 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:30:49,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:49,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:30:50,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:51,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:53,869 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:55,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:30:57,871 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:00,625 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:31:00,626 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:31:04,568 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:31:09,237 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:31:09,237 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:31:09,238 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:31:09,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:09,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:31:10,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:11,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:13,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:15,758 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:31:15,758 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:31:15,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:17,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:29,369 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:31:29,370 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:31:29,370 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:31:29,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:29,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:31:30,932 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:31:30,933 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:31:31,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:32,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:34,639 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:31:34,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:36,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:38,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:46,065 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:31:46,066 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:31:49,475 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:31:49,475 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:31:49,476 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:31:49,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:31:50,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:51,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:52,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:54,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:56,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:31:58,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:01,200 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:32:01,200 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:32:04,711 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:32:09,570 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:32:09,571 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:32:09,571 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:32:09,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:32:10,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:11,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:12,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:14,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:16,332 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:32:16,332 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:32:16,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:18,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:29,680 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:32:29,681 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:32:29,682 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:32:29,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:32:30,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:31,839 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:32:31,840 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:32:31,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:32,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:34,780 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:32:34,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:36,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:38,918 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:46,974 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:32:46,975 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:32:49,799 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:32:49,799 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:32:49,800 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:32:49,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:32:50,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:51,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:52,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:54,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:56,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:32:58,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:02,107 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:33:02,107 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:33:04,855 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:33:09,835 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:33:09,835 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:33:09,836 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:33:09,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:33:10,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:11,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:12,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:14,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:16,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:17,240 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:33:17,240 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:33:18,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:29,937 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:33:29,937 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:33:29,938 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:33:29,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:33:30,942 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:31,942 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:32,382 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:33:32,382 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:33:32,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:34,930 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:33:34,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:36,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:38,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:47,527 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:33:47,527 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:33:50,011 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:33:50,012 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:33:50,012 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:33:50,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:50,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:33:51,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:52,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:55,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:57,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:33:59,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:02,662 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:34:02,662 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:34:05,003 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:34:10,072 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:34:10,073 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:34:10,073 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:34:10,960 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:34:11,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:13,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:15,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:17,798 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:34:17,799 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:34:17,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:19,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:30,189 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:34:30,189 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:34:30,190 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:34:30,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:34:31,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:32,931 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:34:32,931 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:34:33,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:35,077 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:34:35,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:37,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:48,069 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:34:48,069 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:34:50,298 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:34:50,298 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:34:50,299 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:34:50,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:34:51,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:53,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:55,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:34:57,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:03,200 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:35:03,200 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:35:05,153 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:35:10,391 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:35:10,391 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:35:10,391 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:35:10,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:35:11,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:12,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:13,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:15,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:17,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:18,330 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:35:18,330 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:35:30,543 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:35:30,544 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:35:30,544 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:35:30,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:35:31,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:32,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:33,532 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:35:33,532 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:35:34,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:35,225 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:35:36,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:38,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:48,733 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:35:48,734 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:35:50,634 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:35:50,635 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:35:50,635 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:35:51,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:35:53,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:55,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:57,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:35:59,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:03,865 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:36:03,865 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:36:05,299 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:36:10,778 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:36:10,778 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:36:10,779 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:36:11,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:36:13,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:15,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:17,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:18,999 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:36:18,999 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:36:19,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:30,930 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:36:30,931 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:36:30,931 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:36:31,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:36:33,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:34,159 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:36:34,160 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:36:35,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:35,371 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:36:37,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:39,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:49,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:36:49,353 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:36:51,046 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:36:51,046 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:36:51,047 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:36:52,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:36:53,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:55,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:57,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:36:59,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:04,484 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:37:04,484 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:37:05,444 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:37:11,116 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:37:11,117 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:37:11,117 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:37:12,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:37:13,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:15,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:17,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:19,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:19,616 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:37:19,616 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:37:31,197 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:37:31,197 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:37:31,198 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:37:32,053 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:37:33,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:34,788 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:37:34,789 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:37:35,516 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:37:36,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:38,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:40,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:49,956 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:37:49,957 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:37:51,306 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:37:51,307 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:37:51,307 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:37:52,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:37:54,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:56,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:37:58,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:00,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:05,087 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:38:05,088 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:38:05,590 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:38:11,423 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:38:11,424 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:38:11,424 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:38:12,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:38:14,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:16,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:18,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:20,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:20,217 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:38:20,217 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:38:31,493 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:38:31,493 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:38:31,493 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:38:32,081 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:38:34,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:35,428 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:38:35,428 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:38:35,664 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:38:36,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:38,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:40,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:50,565 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:38:50,566 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:38:51,572 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:38:51,572 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:38:51,572 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:38:52,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:38:54,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:56,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:38:58,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:00,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:05,698 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:39:05,699 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:39:05,828 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:39:11,659 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:39:11,659 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:39:11,660 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:39:12,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:39:14,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:16,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:18,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:20,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:20,829 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:39:20,830 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:39:31,728 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:39:31,729 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:39:31,729 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:39:32,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:39:34,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:35,813 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:39:35,989 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:39:35,989 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:39:36,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:38,111 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:40,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:51,125 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:39:51,125 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:39:51,822 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:39:51,823 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:39:51,823 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:39:52,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:39:54,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:57,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:39:59,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:01,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:05,888 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:40:06,257 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:40:06,257 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:40:11,914 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:40:11,915 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:40:11,917 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:40:12,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:40:14,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:15,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:17,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:19,129 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:21,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:21,386 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:40:21,387 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:40:32,007 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:40:32,007 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:40:32,008 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:40:32,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:40:34,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:35,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:35,961 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:40:36,548 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:40:36,548 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:40:37,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:39,138 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:41,139 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:51,735 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:40:51,735 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:40:52,086 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:40:52,087 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:40:52,089 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:40:52,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:40:53,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:54,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:55,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:57,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:40:59,148 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:01,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:06,036 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:41:06,874 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:41:06,874 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:41:12,198 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:41:12,199 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:41:12,199 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:41:13,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:13,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:41:14,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:15,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:17,156 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:19,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:21,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:22,004 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:41:22,004 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:41:32,361 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:41:32,361 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:41:32,364 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:41:33,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:33,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:41:34,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:35,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:36,113 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:41:37,137 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:41:37,137 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:41:37,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:39,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:41,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:52,269 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:41:52,270 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:41:52,498 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:41:52,498 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:41:52,499 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:41:53,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:53,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:41:55,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:57,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:41:59,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:01,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:06,187 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:42:07,410 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:42:07,410 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:42:12,636 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:42:12,636 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:42:12,637 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:42:13,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:13,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:42:15,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:16,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:18,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:20,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:22,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:22,539 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:42:22,540 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:42:32,692 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:42:32,693 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:42:32,693 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:42:33,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:42:34,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:35,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:36,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:36,262 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:42:37,672 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:42:37,672 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:42:38,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:40,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:42,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:52,804 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:42:52,804 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:42:52,939 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:42:52,939 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:42:52,940 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:42:53,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:42:54,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:55,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:56,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:42:58,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:00,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:02,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:06,335 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:43:07,943 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:43:07,944 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:43:12,941 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:43:12,941 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:43:12,942 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:43:13,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:43:14,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:15,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:16,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:18,217 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:20,218 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:22,219 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:23,073 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:43:23,073 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:43:33,068 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:43:33,069 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:43:33,069 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:43:33,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:43:34,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:35,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:36,226 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:36,413 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:43:38,227 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:38,255 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:43:38,255 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:43:40,228 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:42,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:53,224 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:43:53,224 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:43:53,225 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:43:53,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:43:53,392 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:43:53,392 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:43:54,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:55,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:56,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:43:58,236 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:00,237 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:02,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:06,489 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:44:08,525 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:44:08,525 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:44:13,372 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:44:13,372 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:44:13,373 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:44:14,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:14,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:44:15,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:16,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:18,246 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:21,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:23,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:23,658 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:44:23,659 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:44:33,479 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:44:33,479 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:44:33,479 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:44:34,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:44:35,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:36,564 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:44:37,255 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:38,833 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:44:38,833 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:44:39,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:41,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:43,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:53,612 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:44:53,612 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:44:53,613 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:44:54,014 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:44:54,015 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:44:54,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:44:55,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:57,264 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:44:59,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:01,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:03,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:06,639 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:45:09,202 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:45:09,202 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:45:13,699 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:45:13,700 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:45:13,700 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:45:14,272 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:45:15,273 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:17,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:19,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:21,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:23,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:24,332 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:45:24,333 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:45:33,775 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:45:33,775 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:45:33,776 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:45:34,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:45:35,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:36,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:36,713 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:45:37,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:39,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:39,549 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:45:39,550 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:45:41,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:53,850 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:45:53,850 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:45:53,851 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:45:54,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:45:54,708 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:45:54,709 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:45:55,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:56,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:57,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:45:59,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:01,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:06,789 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:46:09,845 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:46:09,846 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:46:13,977 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:46:13,978 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:46:13,978 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:46:14,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:46:15,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:16,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:17,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:19,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:21,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:24,980 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:46:24,980 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:46:34,063 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:46:34,063 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:46:34,064 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:46:34,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:46:35,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:36,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:36,865 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:46:37,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:39,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:40,159 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:46:40,159 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:46:41,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:43,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:54,185 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:46:54,185 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:46:54,186 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:46:54,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:46:55,316 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:46:55,316 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:46:56,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:46:58,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:00,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:02,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:06,943 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:47:10,477 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:47:10,477 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:47:14,265 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:47:14,265 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:47:14,266 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:47:14,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:47:16,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:18,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:20,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:22,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:25,608 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:47:25,608 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:47:34,362 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:47:34,363 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:47:34,363 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:47:35,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:47:36,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:37,020 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:47:38,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:40,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:40,740 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:47:40,740 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:47:42,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:54,453 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:47:54,454 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:47:54,454 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:47:55,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:47:55,941 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:47:55,942 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:47:56,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:47:58,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:00,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:02,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:07,096 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:48:11,079 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:48:11,080 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:48:14,594 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:48:14,595 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:48:14,595 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:48:15,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:48:16,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:18,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:20,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:22,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:26,218 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:48:26,218 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:48:34,677 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:48:34,677 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:48:34,678 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:48:35,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:48:36,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:37,172 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:48:37,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:38,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:40,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:41,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:48:41,354 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:48:42,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:54,769 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:48:54,770 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:48:54,770 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:48:55,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:48:56,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:56,516 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:48:56,516 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:48:57,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:48:58,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:00,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:02,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:07,249 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:49:11,651 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:49:11,651 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:49:14,930 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:49:14,930 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:49:14,931 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:49:15,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:49:16,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:17,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:18,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:20,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:22,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:26,782 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:49:26,782 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:49:35,075 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:49:35,076 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:49:35,076 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:49:35,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:49:36,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:37,325 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:49:37,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:39,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:41,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:41,915 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:49:41,916 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:49:43,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:55,157 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:49:55,158 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:49:55,159 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:49:55,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:49:57,085 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:49:57,085 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:49:57,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:49:59,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:01,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:03,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:07,400 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:50:12,277 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:50:12,278 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:50:15,278 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:50:15,279 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:50:15,280 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:50:15,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:50:17,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:19,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:21,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:23,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:27,409 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:50:27,409 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:50:35,341 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:50:35,342 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:50:35,342 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:50:35,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:50:37,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:37,478 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:50:39,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:41,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:42,540 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:50:42,541 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:50:43,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:55,418 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:50:55,418 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:50:55,419 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:50:55,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:50:57,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:50:57,745 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:50:57,745 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:50:59,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:01,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:03,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:07,554 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:51:12,948 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:51:12,948 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:51:15,526 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:51:15,526 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:51:15,527 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:51:16,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:51:17,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:19,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:21,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:23,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:28,079 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:51:28,080 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:51:35,658 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:51:35,658 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:51:35,659 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:51:36,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:51:37,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:37,629 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:51:39,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:41,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:43,209 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:51:43,210 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:51:43,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:55,762 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:51:55,762 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:51:55,763 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:51:56,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:51:57,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:51:58,341 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:51:58,341 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:51:59,462 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:01,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:03,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:07,705 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:52:13,476 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:52:13,476 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:52:15,864 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:52:15,864 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:52:15,865 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:52:16,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:52:17,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:19,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:21,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:23,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:28,606 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:52:28,606 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:52:35,999 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:52:35,999 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:52:36,000 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:52:36,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:52:37,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:37,783 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:52:38,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:40,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:42,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:43,738 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:52:43,738 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:52:44,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:56,159 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:52:56,160 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:52:56,160 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:52:56,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:52:57,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:58,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:52:58,929 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:52:58,930 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:53:00,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:02,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:04,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:07,852 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:53:14,072 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:53:14,072 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:53:16,217 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:53:16,218 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:53:16,218 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:53:16,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:53:18,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:20,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:22,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:24,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:29,201 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:53:29,202 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:53:36,296 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:53:36,296 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:53:36,297 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:53:36,504 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:53:37,923 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:53:38,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:40,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:42,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:44,332 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:53:44,332 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:53:44,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:56,391 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:53:56,392 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:53:56,392 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:53:56,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:53:58,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:53:59,492 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:53:59,492 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:54:03,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:54:05,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:54:07,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:54:07,997 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:54:10,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:54:15,376 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:54:15,398 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status