diff --git "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" --- "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" +++ "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" @@ -19421,3 +19421,3628 @@ 2021-07-16 18:54:37,775 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log 2021-07-16 18:54:39,776 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log 2021-07-16 18:54:41,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:54:44,252 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:54:50,501 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:54:50,501 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:54:54,176 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:54:54,177 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:54:54,177 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:54:54,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:54:55,783 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:54:57,784 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:54:59,785 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:01,785 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:05,644 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:55:05,645 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:55:14,216 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:55:14,216 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:55:14,216 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:55:14,324 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:55:14,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:55:15,791 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:17,792 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:19,793 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:20,780 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:55:20,780 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:55:21,794 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:34,255 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:55:34,256 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:55:34,256 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:55:34,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:55:35,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:35,911 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:55:35,911 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:55:37,800 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:39,801 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:41,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:43,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:44,394 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:55:51,048 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:55:51,049 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:55:54,293 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:55:54,294 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:55:54,294 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:55:54,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:55:55,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:57,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:55:59,809 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:01,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:03,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:06,181 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:56:06,182 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:56:14,335 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:56:14,335 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:56:14,335 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:56:14,465 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:56:14,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:56:15,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:17,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:19,818 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:21,329 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:56:21,330 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:56:21,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:23,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:34,372 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:56:34,372 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:56:34,373 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:56:34,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:56:35,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:36,463 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:56:36,464 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:56:37,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:39,827 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:41,827 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:43,828 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:44,539 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:56:51,596 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:56:51,596 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:56:54,409 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:56:54,410 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:56:54,410 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:56:54,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:56:55,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:57,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:56:59,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:01,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:03,836 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:06,734 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:57:06,734 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:57:14,448 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:57:14,448 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:57:14,449 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:57:14,613 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:57:14,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:57:15,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:17,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:19,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:21,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:21,864 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:57:21,864 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:57:23,845 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:34,491 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:57:34,492 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:57:34,492 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:57:34,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:57:35,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:37,005 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:57:37,006 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:57:37,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:39,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:41,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:43,853 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:44,688 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:57:52,138 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:57:52,138 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:57:54,531 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:57:54,531 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:57:54,532 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:57:54,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:57:55,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:57,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:57:59,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:01,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:03,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:07,268 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:58:07,269 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:58:14,572 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:58:14,572 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:58:14,573 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:58:14,764 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:58:14,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:58:15,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:17,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:19,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:21,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:22,402 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:58:22,402 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:58:23,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:34,614 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:58:34,614 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:58:34,615 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:58:34,873 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:58:35,874 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:37,535 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:58:37,535 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:58:37,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:39,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:41,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:43,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:44,840 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:58:52,666 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:58:52,667 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:58:54,657 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:58:54,658 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:58:54,658 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:58:54,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:58:55,882 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:57,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:58:59,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:01,884 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:07,795 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:59:07,795 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:59:14,757 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:59:14,758 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:59:14,758 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:59:14,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:59:14,918 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:59:15,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:17,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:19,892 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:21,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:22,927 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:59:22,928 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:59:23,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:34,805 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:59:34,806 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:59:34,806 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:59:34,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:59:35,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:37,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:38,107 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:59:38,107 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:59:39,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:41,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:43,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:44,996 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 18:59:53,241 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 18:59:53,241 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 18:59:54,851 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 18:59:54,852 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 18:59:54,852 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 18:59:54,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 18:59:55,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:57,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 18:59:59,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:01,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:03,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:08,373 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:00:08,374 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:00:14,896 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:00:14,896 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:00:14,897 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:00:14,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:00:15,073 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:00:15,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:17,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:19,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:21,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:23,522 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:00:23,522 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:00:23,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:34,944 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:00:34,944 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:00:34,945 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:00:35,924 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:00:35,924 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:37,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:38,657 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:00:38,658 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:00:39,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:41,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:43,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:45,150 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:00:53,788 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:00:53,788 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:00:54,984 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:00:54,985 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:00:54,985 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:00:55,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:00:55,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:57,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:00:59,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:01,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:03,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:08,937 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:01:08,937 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:01:15,044 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:01:15,044 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:01:15,045 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:01:15,226 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:01:15,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:01:15,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:18,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:20,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:22,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:24,069 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:01:24,069 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:01:24,947 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:35,087 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:01:35,088 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:01:35,088 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:01:35,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:01:36,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:38,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:39,204 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:01:39,204 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:01:40,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:42,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:44,956 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:45,306 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:01:54,335 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:01:54,336 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:01:55,125 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:01:55,125 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:01:55,125 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:01:55,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:01:56,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:01:58,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:00,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:02,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:04,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:09,465 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:02:09,465 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:02:15,215 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:02:15,216 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:02:15,216 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:02:15,384 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:02:15,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:02:16,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:18,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:20,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:22,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:24,601 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:02:24,602 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:02:24,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:35,254 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:02:35,255 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:02:35,255 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:02:35,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:02:36,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:38,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:39,748 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:02:39,748 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:02:40,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:42,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:44,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:45,462 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:02:54,890 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:02:54,890 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:02:55,295 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:02:55,295 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:02:55,296 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:02:55,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:02:56,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:02:58,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:00,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:02,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:04,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:10,018 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:03:10,018 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:03:15,336 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:03:15,336 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:03:15,337 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:03:15,540 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:03:15,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:03:16,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:18,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:20,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:22,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:24,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:25,152 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:03:25,152 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:03:35,375 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:03:35,375 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:03:35,376 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:03:36,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:03:37,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:39,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:40,291 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:03:40,291 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:03:41,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:43,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:45,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:45,618 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:03:55,413 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:03:55,414 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:03:55,414 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:03:55,453 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:03:55,454 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:03:56,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:03:57,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:03:59,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:01,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:03,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:05,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:10,587 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:04:10,587 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:04:15,453 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:04:15,453 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:04:15,454 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:04:15,696 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:04:16,021 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:04:17,021 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:19,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:21,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:23,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:25,720 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:04:25,720 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:04:35,495 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:04:35,495 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:04:35,496 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:04:36,029 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:04:37,030 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:39,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:40,850 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:04:40,851 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:04:41,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:43,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:45,778 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:04:55,535 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:04:55,536 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:04:55,537 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:04:56,021 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:04:56,022 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:04:56,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:04:57,038 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:04:59,039 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:01,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:03,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:05,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:11,153 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:05:11,153 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:05:15,638 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:05:15,638 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:05:15,639 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:05:15,857 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:05:16,046 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:05:17,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:19,048 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:21,049 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:23,049 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:26,285 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:05:26,286 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:05:35,705 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:05:35,706 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:05:35,706 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:05:36,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:05:37,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:39,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:41,057 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:41,421 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:05:41,422 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:05:43,058 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:45,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:45,933 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:05:55,746 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:05:55,747 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:05:55,747 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:05:56,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:05:56,594 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:05:56,594 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:05:57,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:05:59,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:01,065 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:03,066 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:05,067 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:11,733 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:06:11,733 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:06:15,786 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:06:15,787 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:06:15,787 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:06:16,011 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:06:16,071 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:06:17,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:19,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:21,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:23,074 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:25,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:26,870 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:06:26,870 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:06:35,825 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:06:35,825 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:06:35,826 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:06:36,079 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:06:37,080 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:39,080 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:41,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:42,003 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:06:42,003 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:06:43,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:45,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:46,089 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:06:55,860 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:06:55,861 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:06:55,861 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:06:56,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:06:57,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:06:57,142 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:06:57,142 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:06:59,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:01,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:03,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:05,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:12,278 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:07:12,278 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:07:15,900 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:07:15,901 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:07:15,901 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:07:16,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:07:16,167 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:07:17,097 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:19,097 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:21,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:23,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:25,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:27,490 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:07:27,490 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:07:35,942 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:07:35,943 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:07:35,943 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:07:36,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:07:37,105 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:39,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:41,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:42,674 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:07:42,675 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:07:43,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:45,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:46,244 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:07:55,982 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:07:55,983 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:07:55,983 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:07:56,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:07:57,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:07:57,829 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:07:57,829 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:07:59,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:01,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:03,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:05,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:12,964 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:08:12,965 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:08:16,020 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:08:16,021 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:08:16,021 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:08:16,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:08:16,322 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:08:17,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:19,122 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:21,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:23,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:25,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:28,098 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:08:28,098 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:08:36,069 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:08:36,070 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:08:36,070 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:08:36,129 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:08:37,129 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:39,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:41,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:43,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:43,230 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:08:43,230 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:08:46,399 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:08:56,107 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:08:56,108 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:08:56,108 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:08:56,137 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:08:57,138 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:08:58,379 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:08:58,379 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:08:59,138 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:01,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:03,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:13,517 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:09:13,518 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:09:16,167 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:09:16,167 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:09:16,167 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:09:16,477 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:09:17,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:09:17,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:19,147 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:21,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:23,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:25,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:28,650 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:09:28,650 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:09:36,209 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:09:36,209 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:09:36,210 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:09:37,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:09:37,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:39,155 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:41,155 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:43,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:43,783 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:09:43,784 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:09:45,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:46,555 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:09:56,250 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:09:56,250 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:09:56,251 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:09:57,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:09:57,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:09:58,915 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:09:58,916 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:10:00,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:02,164 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:04,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:06,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:14,046 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:10:14,046 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:10:16,291 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:10:16,291 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:10:16,292 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:10:16,633 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:10:17,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:10:18,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:20,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:22,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:24,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:26,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:29,180 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:10:29,180 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:10:36,330 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:10:36,331 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:10:36,331 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:10:37,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:10:38,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:40,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:42,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:44,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:44,311 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:10:44,311 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:10:46,711 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:10:56,370 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:10:56,370 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:10:56,371 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:10:57,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:10:58,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:10:59,446 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:10:59,446 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:11:00,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:02,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:04,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:14,587 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:11:14,587 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:11:16,410 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:11:16,411 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:11:16,411 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:11:16,789 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:11:17,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:11:18,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:20,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:22,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:24,200 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:26,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:29,719 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:11:29,719 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:11:36,450 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:11:36,450 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:11:36,451 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:11:37,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:11:38,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:40,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:42,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:44,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:44,858 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:11:44,859 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:11:46,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:11:46,869 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:11:56,486 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:11:56,487 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:11:56,487 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:11:57,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:11:58,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:00,004 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:12:00,004 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:12:00,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:02,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:04,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:06,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:15,140 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:12:15,140 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:12:16,524 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:12:16,524 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:12:16,525 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:12:16,949 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:12:17,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:12:18,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:20,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:22,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:24,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:26,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:30,273 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:12:30,274 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:12:36,587 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:12:36,587 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:12:36,587 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:12:37,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:12:38,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:40,234 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:42,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:44,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:45,406 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:12:45,407 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:12:46,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:12:47,026 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:12:56,626 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:12:56,626 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:12:56,627 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:12:57,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:12:58,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:00,243 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:00,538 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:13:00,539 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:13:02,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:04,245 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:06,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:15,675 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:13:15,675 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:13:16,661 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:13:16,662 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:13:16,662 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:13:17,104 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:13:17,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:13:18,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:20,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:22,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:24,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:26,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:30,810 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:13:30,810 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:13:36,699 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:13:36,699 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:13:36,700 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:13:37,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:13:38,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:40,261 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:42,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:44,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:13:45,943 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:13:45,943 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:13:47,182 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:13:56,738 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:13:56,739 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:13:56,739 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:13:57,269 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:13:58,269 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:00,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:01,079 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:14:01,079 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:14:02,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:04,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:16,234 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:14:16,234 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:14:16,777 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:14:16,777 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:14:16,778 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:14:17,258 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:14:17,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:14:18,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:20,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:22,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:24,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:26,281 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:31,364 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:14:31,365 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:14:36,815 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:14:36,816 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:14:36,816 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:14:37,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:14:38,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:40,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:42,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:44,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:46,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:14:46,495 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:14:46,495 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:14:47,337 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:14:56,871 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:14:56,871 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:14:56,872 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:14:57,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:14:58,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:00,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:01,625 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:15:01,626 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:15:02,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:04,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:06,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:16,763 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:15:16,764 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:15:16,911 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:15:16,912 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:15:16,912 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:15:17,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:15:17,415 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:15:18,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:20,307 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:22,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:24,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:31,893 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:15:31,894 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:15:36,955 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:15:36,956 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:15:36,956 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:15:37,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:15:38,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:40,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:42,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:44,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:15:47,042 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:15:47,042 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:15:47,492 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:15:56,995 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:15:56,996 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:15:56,996 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:15:57,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:15:58,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:00,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:02,176 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:16:02,176 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:16:02,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:04,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:06,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:17,036 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:16:17,037 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:16:17,037 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:16:17,314 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:16:17,314 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:16:17,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:16:17,569 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:16:18,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:20,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:22,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:24,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:26,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:32,473 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:16:32,473 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:16:37,079 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:16:37,080 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:16:37,080 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:16:37,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:16:38,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:40,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:42,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:44,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:46,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:16:47,603 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:16:47,603 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:16:47,734 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:16:57,119 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:16:57,120 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:16:57,120 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:16:57,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:16:58,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:00,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:02,349 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:02,736 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:17:02,736 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:17:04,350 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:06,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:17,157 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:17:17,158 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:17:17,158 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:17:17,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:17:17,725 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:17:17,901 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:17:17,901 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:17:18,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:20,356 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:22,357 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:24,358 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:26,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:33,041 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:17:33,041 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:17:37,195 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:17:37,196 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:17:37,196 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:17:37,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:17:38,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:40,365 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:42,366 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:44,367 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:46,368 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:17:47,801 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:17:48,175 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:17:48,176 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:17:57,235 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:17:57,236 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:17:57,238 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:17:57,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:17:58,373 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:00,374 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:02,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:03,322 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:18:03,322 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:18:04,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:06,377 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:17,274 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:18:17,275 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:18:17,275 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:18:17,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:18:17,871 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:18:18,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:18,482 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:18:18,483 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:18:20,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:22,383 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:24,384 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:33,620 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:18:33,621 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:18:37,315 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:18:37,315 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:18:37,318 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:18:37,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:18:38,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:40,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:42,391 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:44,392 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:18:47,944 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:18:48,754 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:18:48,754 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:18:57,355 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:18:57,355 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:18:57,355 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:18:57,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:18:58,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:00,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:02,399 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:03,886 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:19:03,886 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:19:04,400 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:17,393 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:19:17,393 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:19:17,395 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:19:17,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:19:18,013 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:19:19,019 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:19:19,019 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:19:19,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:21,407 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:23,408 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:25,409 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:27,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:34,157 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:19:34,157 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:19:37,434 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:19:37,434 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:19:37,435 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:19:38,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:19:39,415 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:41,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:43,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:45,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:47,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:19:48,082 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:19:49,290 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:19:49,290 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:19:57,500 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:19:57,501 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:19:57,501 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:19:58,424 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:19:59,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:01,426 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:03,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:04,422 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:20:04,423 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:20:05,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:07,429 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:17,539 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:20:17,539 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:20:17,540 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:20:18,151 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:20:18,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:20:19,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:19,573 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:20:19,574 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:20:21,435 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:23,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:25,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:27,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:34,711 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:20:34,712 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:20:37,591 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:20:37,591 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:20:37,592 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:20:38,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:20:39,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:41,444 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:43,444 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:45,445 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:20:48,223 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:20:49,849 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:20:49,849 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:20:57,629 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:20:57,630 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:20:57,630 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:20:58,450 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:20:59,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:01,452 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:03,453 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:04,980 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:21:04,980 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:21:05,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:17,669 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:21:17,669 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:21:17,670 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:21:18,293 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:21:18,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:21:19,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:20,119 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:21:20,119 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:21:21,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:23,461 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:25,462 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:35,253 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:21:35,253 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:21:37,706 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:21:37,706 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:21:37,707 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:21:38,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:21:39,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:41,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:43,469 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:45,470 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:21:48,363 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:21:50,385 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:21:50,386 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:21:57,743 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:21:57,744 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:21:57,744 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:21:58,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:21:59,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:01,477 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:03,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:05,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:05,516 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:22:05,516 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:22:17,780 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:22:17,780 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:22:17,781 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:22:18,434 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:22:18,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:22:19,484 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:20,680 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:22:20,680 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:22:21,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:23,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:25,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:35,810 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:22:35,810 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:22:37,818 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:22:37,818 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:22:37,819 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:22:38,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:22:39,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:41,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:43,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:45,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:47,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:22:48,505 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:22:50,944 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:22:50,944 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:22:57,856 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:22:57,856 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:22:57,857 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:22:58,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:22:59,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:01,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:03,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:05,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:06,074 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:23:06,075 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:23:07,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:17,894 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:23:17,894 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:23:17,895 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:23:18,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:23:18,576 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:23:19,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:21,208 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:23:21,208 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:23:21,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:23,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:25,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:27,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:36,353 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:23:36,353 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:23:37,930 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:23:37,931 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:23:37,931 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:23:38,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:23:39,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:41,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:43,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:45,521 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:47,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:23:48,654 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:23:51,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:23:51,486 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:23:57,968 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:23:57,969 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:23:57,969 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:23:58,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:23:59,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:01,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:03,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:05,530 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:06,619 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:24:06,619 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:24:07,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:18,008 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:24:18,008 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:24:18,009 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:24:18,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:24:18,728 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:24:19,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:21,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:21,751 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:24:21,751 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:24:23,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:25,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:27,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:36,885 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:24:36,886 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:24:38,048 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:24:38,048 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:24:38,049 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:24:38,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:24:39,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:41,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:43,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:45,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:47,548 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:24:48,805 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:24:52,019 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:24:52,020 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:24:58,085 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:24:58,085 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:24:58,086 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:24:58,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:24:59,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:01,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:03,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:05,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:07,151 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:25:07,151 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:25:18,129 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:25:18,129 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:25:18,130 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:25:18,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:25:18,884 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:25:19,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:21,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:22,284 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:25:22,284 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:25:23,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:25,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:37,415 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:25:37,415 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:25:38,167 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:25:38,167 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:25:38,167 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:25:38,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:25:39,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:41,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:43,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:45,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:25:48,961 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:25:52,547 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:25:52,548 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:25:58,235 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:25:58,235 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:25:58,236 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:25:58,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:25:59,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:01,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:03,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:05,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:07,678 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:26:07,678 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:26:18,272 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:26:18,272 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:26:18,273 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:26:18,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:26:19,035 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:26:19,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:21,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:22,816 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:26:22,817 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:26:23,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:25,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:37,947 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:26:37,947 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:26:38,312 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:26:38,312 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:26:38,313 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:26:38,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:26:39,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:41,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:43,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:45,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:26:49,111 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:26:53,079 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:26:53,080 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:26:58,348 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:26:58,348 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:26:58,349 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:26:58,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:26:59,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:01,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:03,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:05,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:07,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:08,208 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:27:08,208 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:27:18,388 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:27:18,388 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:27:18,389 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:27:18,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:27:19,189 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:27:19,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:21,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:23,340 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:27:23,341 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:27:23,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:25,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:27,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:38,425 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:27:38,425 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:27:38,426 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:27:38,484 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:27:38,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:27:38,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:27:39,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:41,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:43,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:45,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:47,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:27:49,267 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:27:53,626 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:27:53,626 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:27:58,462 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:27:58,462 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:27:58,463 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:27:58,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:27:59,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:01,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:03,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:05,628 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:07,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:08,766 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:28:08,767 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:28:18,500 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:28:18,500 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:28:18,501 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:28:18,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:28:19,347 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:28:19,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:21,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:23,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:23,898 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:28:23,898 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:28:25,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:38,536 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:28:38,536 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:28:38,536 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:28:38,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:28:39,028 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:28:39,028 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:28:39,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:41,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:43,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:45,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:28:49,424 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:28:54,163 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:28:54,164 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:28:58,588 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:28:58,589 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:28:58,589 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:28:58,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:28:59,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:00,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:01,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:03,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:05,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:07,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:09,295 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:29:09,296 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:29:18,627 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:29:18,628 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:29:18,629 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:29:18,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:29:19,500 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:29:20,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:22,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:24,432 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:29:24,433 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:29:24,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:26,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:28,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:38,672 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:29:38,673 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:29:38,673 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:29:39,569 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:29:39,569 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:29:39,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:29:40,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:42,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:44,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:46,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:48,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:29:49,578 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:29:54,726 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:29:54,726 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:29:58,740 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:29:58,741 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:29:58,741 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:29:59,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:30:00,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:02,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:04,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:06,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:08,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:09,856 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:30:09,856 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:30:18,779 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:30:18,780 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:30:18,780 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:30:19,657 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:30:19,681 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:30:20,681 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:22,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:24,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:24,987 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:30:24,988 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:30:26,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:28,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:38,848 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:30:38,848 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:30:38,849 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:30:39,689 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:30:40,129 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:30:40,129 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:30:40,689 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:42,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:44,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:46,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:48,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:30:49,734 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:30:55,264 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:30:55,264 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:30:58,887 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:30:58,887 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:30:58,888 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:30:59,697 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:31:00,697 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:02,698 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:04,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:06,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:08,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:10,396 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:31:10,397 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:31:18,945 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:31:18,945 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:31:18,946 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:31:19,705 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:31:19,812 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:31:20,705 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:22,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:24,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:25,529 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:31:25,529 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:31:26,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:28,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:38,984 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:31:38,984 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:31:38,985 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:31:39,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:31:40,707 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:31:40,708 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:31:40,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:42,714 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:44,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:46,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:31:49,890 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:31:55,839 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:31:55,840 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:31:59,024 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:31:59,025 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:31:59,025 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:31:59,722 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:32:00,722 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:02,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:04,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:06,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:08,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:10,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:10,972 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:32:10,973 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:32:12,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:14,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:16,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:18,730 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:19,967 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:32:20,731 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:22,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:24,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:26,128 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:32:26,129 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:32:26,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:28,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:30,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:32,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:34,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:36,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:38,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:40,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:41,266 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:32:41,266 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:32:42,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:44,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:46,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:48,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:50,045 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:32:50,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:52,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:54,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:56,397 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:32:56,398 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:32:56,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:32:58,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:00,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:02,973 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:33:02,973 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:33:02,973 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:33:03,750 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:33:04,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:06,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:08,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:10,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:11,527 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:33:11,528 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:33:20,122 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:33:23,018 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:33:23,018 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:33:23,019 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:33:23,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:33:24,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:26,663 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:33:26,663 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:33:26,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:28,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:30,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:41,793 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:33:41,794 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:33:43,059 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:33:43,059 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:33:43,060 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:33:43,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:33:44,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:45,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:47,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:49,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:50,200 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:33:51,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:33:56,927 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:33:56,927 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:34:03,097 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:34:03,097 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:34:03,097 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:34:03,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:34:04,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:05,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:07,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:09,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:11,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:12,058 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:34:12,058 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:34:20,305 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:34:23,135 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:34:23,135 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:34:23,136 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:34:23,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:34:24,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:25,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:27,196 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:34:27,197 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:34:27,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:29,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:31,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:42,326 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:34:42,327 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:34:43,174 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:34:43,174 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:34:43,175 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:34:43,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:34:44,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:45,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:47,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:49,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:50,381 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:34:51,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:34:57,458 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:34:57,459 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:35:03,209 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:35:03,210 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:35:03,210 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:35:03,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:35:04,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:05,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:07,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:09,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:11,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:12,591 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:35:12,592 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:35:20,457 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:35:23,258 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:35:23,259 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:35:23,259 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:35:23,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:35:25,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:27,751 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:35:27,752 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:35:27,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:29,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:31,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:43,297 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:35:43,297 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:35:43,298 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:35:43,592 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:35:43,592 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:35:43,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:35:45,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:47,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:49,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:50,532 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:35:51,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:35:58,731 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:35:58,732 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:36:03,337 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:36:03,337 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:36:03,338 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:36:03,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:36:05,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:07,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:09,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:11,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:13,868 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:36:13,869 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:36:20,608 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:36:23,374 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:36:23,374 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:36:23,375 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:36:23,838 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:36:25,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:27,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:29,005 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:36:29,006 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:36:29,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:31,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:43,422 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:36:43,423 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:36:43,423 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:36:43,847 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:36:44,300 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:36:44,300 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:36:45,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:47,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:49,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:50,687 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:36:51,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:36:59,476 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:36:59,476 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:37:03,475 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:37:03,476 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:37:03,476 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:37:03,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:37:05,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:07,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:09,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:11,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:14,608 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:37:14,608 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:37:20,768 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:37:23,512 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:37:23,512 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:37:23,513 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:37:23,864 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:37:25,865 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:27,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:29,860 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:37:29,860 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:37:29,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:31,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:43,569 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:37:43,570 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:37:43,570 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:37:43,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:37:45,022 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:37:45,023 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:37:45,873 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:47,873 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:49,874 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:37:50,847 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:37:51,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:00,462 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:38:00,462 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:38:03,608 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:38:03,608 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:38:03,609 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:38:03,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:38:05,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:07,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:09,882 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:11,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:15,594 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:38:15,595 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:38:20,924 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:38:23,644 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:38:23,645 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:38:23,645 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:38:23,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:38:25,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:27,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:29,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:30,726 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:38:30,727 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:38:31,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:43,684 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:38:43,684 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:38:43,684 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:38:43,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:38:45,871 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:38:45,872 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:38:45,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:47,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:49,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:38:51,002 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:38:51,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:01,007 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:39:01,007 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:39:03,721 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:39:03,721 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:39:03,722 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:39:03,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:39:05,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:07,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:09,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:11,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:16,139 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:39:16,139 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:39:21,080 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:39:23,758 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:39:23,758 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:39:23,759 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:39:23,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:39:25,914 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:27,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:29,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:31,269 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:39:31,270 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:39:31,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:43,795 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:39:43,796 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:39:43,796 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:39:43,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:39:45,922 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:46,402 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:39:46,402 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:39:47,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:49,924 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:39:51,158 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:39:51,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:01,533 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:40:01,533 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:40:03,832 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:40:03,832 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:40:03,833 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:40:03,930 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:40:05,931 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:07,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:09,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:11,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:16,670 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:40:16,671 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:40:21,236 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:40:23,889 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:40:23,890 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:40:23,890 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:40:23,940 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:40:25,941 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:27,942 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:29,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:31,800 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:40:31,801 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:40:31,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:43,926 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:40:43,926 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:40:43,927 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:40:43,949 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:40:45,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:46,932 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:40:46,933 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:40:47,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:49,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:40:51,314 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:40:51,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:02,062 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:41:02,062 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:41:03,962 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:41:03,962 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:41:03,963 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:41:04,958 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:41:05,958 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:07,959 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:09,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:11,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:17,191 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:41:17,192 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:41:21,391 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:41:24,001 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:41:24,001 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:41:24,001 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:41:24,966 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:41:25,967 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:27,967 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:29,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:31,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:32,322 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:41:32,323 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:41:44,039 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:41:44,040 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:41:44,040 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:41:44,975 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:41:45,975 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:46,976 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:47,461 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:41:47,461 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:41:48,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:50,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:41:51,468 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:41:52,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:02,602 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:42:02,602 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:42:04,078 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:42:04,078 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:42:04,079 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:42:04,984 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:42:05,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:06,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:08,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:10,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:12,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:17,734 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:42:17,734 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:42:21,548 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:42:24,118 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:42:24,118 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:42:24,119 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:42:24,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:42:25,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:26,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:28,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:30,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:32,866 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:42:32,866 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:42:32,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:44,155 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:42:44,156 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:42:44,156 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:42:45,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:42:46,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:47,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:47,999 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:42:48,000 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:42:49,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:51,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:42:51,626 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:42:53,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:03,129 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:43:03,130 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:43:04,196 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:43:04,197 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:43:04,197 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:43:05,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:43:06,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:07,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:09,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:11,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:13,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:18,271 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:43:18,271 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:43:21,703 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:43:24,244 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:43:24,244 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:43:24,245 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:43:25,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:43:26,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:27,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:29,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:31,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:33,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:33,402 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:43:33,402 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:43:44,286 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:43:44,287 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:43:44,287 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:43:45,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:43:46,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:47,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:48,537 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:43:48,538 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:43:49,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:51,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:43:51,780 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:43:53,036 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:03,672 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:44:03,672 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:44:04,338 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:44:04,338 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:44:04,339 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:44:05,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:44:07,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:09,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:11,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:13,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:18,806 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:44:18,807 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:44:21,858 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:44:24,376 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:44:24,376 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:44:24,377 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:44:25,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:44:27,051 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:29,052 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:31,053 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:33,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:33,934 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:44:33,935 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:44:44,413 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:44:44,413 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:44:44,414 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:44:45,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:44:47,060 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:49,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:49,070 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:44:49,070 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:44:51,062 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:44:51,936 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:44:53,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:04,205 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:45:04,205 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:45:04,450 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:45:04,450 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:45:04,451 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:45:05,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:45:07,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:09,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:11,070 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:13,071 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:19,338 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:45:19,338 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:45:22,014 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:45:24,486 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:45:24,487 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:45:24,487 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:45:25,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:45:27,077 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:29,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:31,079 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:33,080 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:34,470 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:45:34,471 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:45:44,527 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:45:44,528 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:45:44,528 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:45:45,085 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:45:47,085 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:49,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:49,604 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:45:49,604 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:45:51,087 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:45:52,091 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:45:53,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:04,565 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:46:04,566 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:46:04,566 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:46:04,739 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:46:04,739 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:46:05,092 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:46:07,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:09,094 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:11,095 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:13,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:19,879 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:46:19,880 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:46:22,169 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:46:24,606 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:46:24,606 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:46:24,607 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:46:25,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:46:27,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:29,102 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:31,103 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:33,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:35,011 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:46:35,011 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:46:44,643 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:46:44,643 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:46:44,644 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:46:45,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:46:47,110 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:49,111 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:50,142 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:46:50,143 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:46:51,111 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:46:52,246 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:46:53,112 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:04,683 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:47:04,683 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:47:04,684 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:47:05,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:47:05,299 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:47:05,300 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:47:07,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:09,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:11,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:13,120 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:20,443 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:47:20,444 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:47:22,324 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:47:24,720 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:47:24,721 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:47:24,721 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:47:25,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:47:27,126 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:29,127 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:31,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:33,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:35,575 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:47:35,575 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:47:44,758 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:47:44,758 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:47:44,759 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:47:45,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:47:47,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:49,135 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:50,708 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:47:50,709 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:47:51,135 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:47:52,402 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:47:53,136 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:04,794 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:48:04,795 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:48:04,795 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:48:05,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:48:05,885 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:48:05,886 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:48:07,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:08,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:10,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:12,144 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:14,145 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:21,018 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:48:21,018 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:48:22,479 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:48:24,842 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:48:24,842 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:48:24,843 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:48:25,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:48:27,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:28,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:30,152 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:32,153 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:34,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:36,149 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:48:36,149 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:48:44,880 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:48:44,881 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:48:44,881 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:48:45,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:48:46,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:47,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:48,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:50,161 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:51,288 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:48:51,289 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:48:52,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:48:52,557 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:48:54,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:04,919 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:49:04,919 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:49:04,920 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:49:05,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:49:06,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:06,468 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:49:06,468 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:49:07,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:08,169 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:10,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:12,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:14,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:21,606 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:49:21,607 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:49:22,633 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:49:24,967 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:49:24,968 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:49:24,968 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:49:25,177 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:49:26,177 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:27,178 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:28,178 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:30,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:32,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:34,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:36,738 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:49:36,739 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:49:45,005 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:49:45,005 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:49:45,006 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:49:45,185 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:49:46,186 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:47,186 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:48,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:50,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:51,878 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:49:51,878 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:49:52,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:49:52,709 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:49:54,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:05,070 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:50:05,070 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:50:05,071 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:50:05,194 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:50:06,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:07,010 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:50:07,010 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:50:07,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:08,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:10,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:12,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:14,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:22,145 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:50:22,145 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:50:22,781 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:50:25,109 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:50:25,110 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:50:25,110 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:50:25,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:50:26,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:27,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:28,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:30,205 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:32,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:34,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:37,280 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:50:37,281 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:50:45,169 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:50:45,169 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:50:45,170 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:50:45,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:50:46,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:47,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:48,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:50,214 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:52,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:50:52,415 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:50:52,416 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:50:52,852 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:50:54,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:05,207 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:51:05,208 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:51:05,208 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:51:05,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:51:06,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:07,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:07,605 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:51:07,606 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:51:08,221 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:10,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:12,223 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:14,223 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:22,744 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:51:22,744 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:51:22,923 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:51:25,270 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:51:25,271 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:51:25,271 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:51:26,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:51:26,229 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:27,229 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:28,229 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:30,230 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:32,231 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:34,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:37,877 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:51:37,877 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:51:45,308 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:51:45,308 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:51:45,308 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:51:46,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:51:46,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:47,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:48,238 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:50,239 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:52,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:51:52,996 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:51:53,007 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:51:53,008 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:51:54,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:05,349 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:52:05,349 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:52:05,350 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:52:06,245 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:52:06,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:08,145 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:52:08,146 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:52:08,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:10,247 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:12,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:14,249 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:23,070 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:52:23,648 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:52:23,649 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:52:25,389 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:52:25,389 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:52:25,390 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:52:26,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:52:26,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:28,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:30,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:32,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:34,257 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:38,776 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:52:38,777 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:52:45,428 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:52:45,429 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:52:45,429 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:52:46,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:52:46,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:48,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:50,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:52,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:52:53,146 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:52:53,910 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:52:53,910 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:52:54,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:05,468 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:53:05,468 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:53:05,468 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:53:06,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:53:07,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:08,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:09,041 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:53:09,042 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:53:09,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:11,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:13,273 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:15,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:23,221 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:53:24,178 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:53:24,179 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:53:25,507 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:53:25,508 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:53:25,508 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:53:26,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:53:27,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:28,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:29,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:31,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:33,281 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:35,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:39,314 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:53:39,315 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:53:45,560 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:53:45,561 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:53:45,561 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:53:46,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:53:47,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:48,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:49,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:51,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:53,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:53:53,297 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:53:54,445 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:53:54,445 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:53:55,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:05,600 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:54:05,601 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:54:05,601 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:54:06,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:54:07,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:08,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:09,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:09,577 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:54:09,577 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:54:11,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:13,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:23,372 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:54:24,710 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:54:24,710 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:54:25,639 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:54:25,639 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:54:25,640 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:54:26,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:54:27,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:28,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:29,307 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:31,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:33,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:35,310 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:39,840 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:54:39,840 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:54:45,676 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:54:45,677 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:54:45,677 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:54:46,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:54:47,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:48,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:49,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:51,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:53,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:54:53,449 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:54:54,970 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:54:54,970 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:54:55,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:05,716 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:55:05,716 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:55:05,717 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:55:06,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:55:07,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:08,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:09,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:10,100 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:55:10,100 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:55:11,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:13,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:15,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:23,527 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:55:25,230 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:55:25,231 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:55:25,752 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:55:25,752 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:55:25,753 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:55:26,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:55:27,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:28,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:29,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:31,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:33,335 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:35,336 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:40,361 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:55:40,361 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:55:45,805 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:55:45,805 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:55:45,806 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:55:46,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:55:47,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:48,343 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:49,343 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:51,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:53,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:53,605 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:55:55,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:55:55,498 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:55:55,499 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:56:05,845 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:56:05,846 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:56:05,846 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:56:06,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:56:07,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:08,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:09,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:10,628 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:56:10,628 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:56:11,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:13,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:15,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:23,682 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:56:25,765 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:56:25,765 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:56:25,899 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:56:25,900 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:56:25,900 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:56:26,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:56:27,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:28,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:29,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:31,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:33,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:35,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:40,894 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:56:40,895 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:56:45,942 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:56:45,942 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:56:45,942 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:56:46,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:56:47,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:48,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:49,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:51,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:53,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:53,759 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:56:55,373 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:56:56,026 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:56:56,027 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:57:05,972 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:57:05,973 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:57:05,973 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:57:06,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:57:07,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:08,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:09,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:11,162 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:57:11,162 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:57:11,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:13,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:15,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:23,839 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:57:26,009 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:57:26,009 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:57:26,010 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:57:26,310 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:57:26,310 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:57:26,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:57:27,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:28,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:29,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:31,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:33,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:41,447 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:57:41,447 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:57:46,045 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:57:46,045 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:57:46,046 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:57:46,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:57:47,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:48,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:49,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:51,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:53,399 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:57:53,916 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:57:56,580 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:57:56,581 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:58:06,083 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:58:06,083 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:58:06,084 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:58:06,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:58:07,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:08,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:09,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:11,407 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:11,712 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:58:11,712 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:58:13,408 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:23,994 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:58:26,124 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:58:26,124 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:58:26,125 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:58:26,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:58:26,853 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:58:26,853 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:58:27,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:28,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:29,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:31,415 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:33,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:42,019 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:58:42,019 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:58:46,161 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:58:46,161 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:58:46,162 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:58:46,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:58:48,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:50,423 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:52,424 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:54,071 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:58:54,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:56,426 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:58:57,153 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:58:57,154 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:59:06,199 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:59:06,200 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:59:06,200 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:59:06,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:59:08,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:10,432 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:12,284 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:59:12,285 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:59:12,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:14,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:24,149 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:59:26,236 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:59:26,236 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:59:26,237 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:59:26,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:59:27,416 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:59:27,417 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:59:28,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:30,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:32,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:34,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:42,554 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:59:42,554 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 19:59:46,272 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 19:59:46,273 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 19:59:46,273 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 19:59:46,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 19:59:48,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:50,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:52,450 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:54,226 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 19:59:54,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 19:59:57,685 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 19:59:57,686 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:00:06,311 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:00:06,311 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:00:06,311 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:00:06,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:00:08,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:10,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:12,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:12,818 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:00:12,819 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:00:14,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:24,304 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:00:26,347 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:00:26,347 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:00:26,348 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:00:26,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:00:27,985 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:00:27,985 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:00:28,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:30,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:32,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:34,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:43,124 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:00:43,124 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:00:46,419 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:00:46,420 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:00:46,420 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:00:46,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:00:48,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:50,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:52,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:54,382 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:00:54,477 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:56,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:00:58,259 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:00:58,260 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:01:06,456 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:01:06,457 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:01:06,457 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:01:06,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:01:08,482 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:10,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:12,484 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:13,433 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:01:13,434 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:01:14,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:24,460 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:01:26,495 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:01:26,496 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:01:26,496 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:01:27,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:01:28,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:28,619 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:01:28,619 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:01:30,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:32,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:34,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:43,856 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:01:43,856 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:01:46,535 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:01:46,535 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:01:46,536 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:01:47,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:01:48,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:50,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:52,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:54,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:01:54,537 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:01:59,091 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:01:59,092 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:02:06,584 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:02:06,585 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:02:06,585 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:02:07,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:02:08,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:10,508 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:12,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:14,275 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:02:14,275 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:02:14,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:24,616 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:02:26,624 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:02:26,625 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:02:26,625 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:02:27,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:02:28,516 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:29,516 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:29,524 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:02:29,524 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:02:30,517 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:32,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:34,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:44,665 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:02:44,665 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:02:46,661 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:02:46,662 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:02:46,662 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:02:47,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:02:48,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:49,525 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:50,525 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:52,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:54,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:02:54,695 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:02:59,799 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:02:59,800 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:03:06,712 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:03:06,712 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:03:06,713 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:03:07,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:03:08,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:09,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:10,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:12,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:14,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:14,931 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:03:14,931 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:03:24,777 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:03:26,759 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:03:26,759 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:03:26,759 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:03:27,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:03:28,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:29,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:30,069 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:03:30,070 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:03:30,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:32,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:34,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:45,203 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:03:45,203 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:03:46,806 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:03:46,807 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:03:46,807 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:03:47,549 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:03:48,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:49,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:50,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:52,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:54,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:03:54,855 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:04:00,333 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:04:00,333 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:04:06,847 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:04:06,848 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:04:06,848 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:04:07,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:04:08,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:09,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:10,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:12,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:14,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:15,464 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:04:15,465 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:04:24,932 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:04:26,906 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:04:26,906 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:04:26,907 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:04:27,566 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:04:28,567 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:29,567 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:30,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:30,615 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:04:30,615 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:04:32,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:34,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:45,749 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:04:45,750 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:04:46,956 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:04:46,957 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:04:46,957 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:04:47,575 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:04:48,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:49,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:50,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:52,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:54,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:04:55,010 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:05:00,884 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:05:00,885 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:05:06,995 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:05:06,995 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:05:06,995 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:05:07,583 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:05:08,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:09,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:10,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:12,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:14,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:16,017 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:05:16,018 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:05:25,087 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:05:27,033 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:05:27,034 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:05:27,034 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:05:27,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:05:28,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:29,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:30,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:31,148 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:05:31,148 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:05:32,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:34,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:46,285 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:05:46,286 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:05:47,072 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:05:47,073 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:05:47,073 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:05:47,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:05:48,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:49,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:50,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:52,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:54,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:05:55,165 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:06:01,425 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:06:01,426 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:06:07,111 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:06:07,111 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:06:07,112 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:06:07,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:06:09,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:11,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:13,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:15,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:16,556 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:06:16,557 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:06:25,243 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:06:27,148 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:06:27,149 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:06:27,149 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:06:27,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:06:29,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:31,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:31,690 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:06:31,690 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:06:33,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:35,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:46,823 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:06:46,823 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:06:47,186 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:06:47,186 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:06:47,187 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:06:47,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:06:49,630 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:51,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:53,632 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:06:55,321 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:06:55,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:01,957 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:07:01,957 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:07:07,227 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:07:07,228 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:07:07,228 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:07:07,638 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:07:09,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:11,640 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:13,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:15,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:17,092 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:07:17,092 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:07:25,399 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:07:27,280 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:07:27,280 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:07:27,280 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:07:27,648 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:07:29,648 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:31,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:32,230 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:07:32,231 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:07:33,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:35,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:47,318 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:07:47,319 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:07:47,319 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:07:47,377 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:07:47,377 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:07:47,656 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:07:49,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:51,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:53,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:07:55,475 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:07:55,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:02,510 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:08:02,510 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:08:07,357 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:08:07,358 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:08:07,358 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:08:07,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:08:09,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:11,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:13,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:15,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:17,640 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:08:17,640 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:08:25,554 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:08:27,436 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:08:27,436 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:08:27,437 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:08:27,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:08:29,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:31,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:32,774 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:08:32,775 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:08:33,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:35,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:47,474 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:08:47,475 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:08:47,475 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:08:47,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:08:47,912 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:08:47,913 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:08:49,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:51,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:53,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:08:55,633 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:08:55,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:03,051 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:09:03,051 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:09:07,514 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:09:07,514 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:09:07,514 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:09:07,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:09:09,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:11,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:13,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:15,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:18,182 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:09:18,182 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:09:25,712 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:09:27,552 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:09:27,552 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:09:27,553 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:09:27,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:09:29,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:31,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:33,315 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:09:33,316 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:09:33,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:35,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:47,589 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:09:47,589 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:09:47,590 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:09:47,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:09:48,473 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:09:48,473 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:09:49,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:51,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:53,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:55,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:09:55,791 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:10:03,614 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:10:03,614 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:10:07,629 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:10:07,629 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:10:07,629 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:10:07,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:10:09,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:11,719 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:13,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:15,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:18,746 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:10:18,747 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:10:25,869 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:10:27,665 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:10:27,666 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:10:27,666 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:10:27,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:10:29,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:31,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:33,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:33,880 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:10:33,881 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:10:35,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:47,703 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:10:47,704 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:10:47,704 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:10:47,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:10:49,036 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:10:49,037 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:10:49,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:51,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:53,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:55,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:10:55,946 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:11:04,174 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:11:04,175 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:11:07,741 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:11:07,742 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:11:07,742 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:11:07,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:11:09,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:11,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:13,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:15,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:19,307 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:11:19,307 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:11:26,024 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:11:27,818 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:11:27,818 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:11:27,819 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:11:28,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:11:29,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:31,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:33,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:34,443 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:11:34,444 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:11:35,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:47,877 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:11:47,877 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:11:47,878 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:11:48,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:11:49,577 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:11:49,577 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:11:49,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:50,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:51,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:53,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:55,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:11:56,102 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:12:04,862 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:12:04,862 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:12:07,916 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:12:07,917 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:12:07,917 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:12:08,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:12:09,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:10,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:11,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:13,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:15,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:20,199 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:12:20,200 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:12:26,175 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:12:27,955 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:12:27,955 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:12:27,956 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:12:28,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:12:29,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:30,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:31,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:33,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:35,330 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:12:35,330 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:12:35,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:47,995 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:12:47,995 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:12:47,996 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:12:48,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:12:49,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:50,498 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:12:50,498 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:12:50,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:51,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:53,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:55,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:12:56,246 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:13:05,630 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:13:05,631 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:13:08,033 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:13:08,034 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:13:08,034 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:13:08,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:13:09,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:10,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:11,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:13,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:15,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:17,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:19,800 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:20,765 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:13:20,765 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:13:21,800 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:23,801 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:25,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:26,317 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:13:27,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:29,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:31,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:33,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:35,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:35,930 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:13:35,931 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:13:38,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:40,809 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:42,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:44,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:46,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:48,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:50,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:51,060 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:13:51,060 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:13:52,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:54,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:56,387 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:13:56,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:13:58,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:00,818 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:02,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:04,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:06,194 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:14:06,194 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:14:06,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:08,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:10,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:11,881 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:14:11,882 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:14:11,882 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:14:12,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:14:14,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:16,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:18,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:20,827 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:21,325 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:14:21,325 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:14:26,462 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:14:31,925 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:14:31,926 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:14:31,926 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:14:32,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:14:32,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:34,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:36,459 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:14:36,459 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:14:36,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:38,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:40,836 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:51,591 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:14:51,592 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:14:51,964 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:14:51,964 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:14:51,965 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:14:52,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:14:52,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:54,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:56,538 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:14:56,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:14:58,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:00,845 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:06,722 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:15:06,723 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:15:12,017 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:15:12,018 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:15:12,018 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:15:12,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:15:12,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:14,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:16,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:18,853 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:20,854 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:22,273 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:15:22,274 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:15:26,616 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:15:32,057 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:15:32,057 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:15:32,058 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:15:32,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:15:32,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:34,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:36,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:37,404 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:15:37,404 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:15:38,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:40,863 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:52,100 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:15:52,100 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:15:52,101 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:15:52,565 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:15:52,565 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:15:52,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:15:52,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:54,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:56,693 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:15:56,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:15:58,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:00,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:07,711 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:16:07,711 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:16:12,141 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:16:12,141 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:16:12,141 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:16:12,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:16:12,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:14,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:16,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:18,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:20,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:22,844 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:16:22,844 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:16:26,772 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:16:32,178 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:16:32,178 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:16:32,178 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:16:32,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:16:32,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:34,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:36,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:37,976 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:16:37,977 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:16:38,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:40,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:52,216 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:16:52,217 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:16:52,217 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:16:52,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:16:52,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:53,113 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:16:53,114 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:16:54,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:56,849 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:16:56,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:16:58,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:00,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:08,249 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:17:08,250 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:17:12,254 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:17:12,255 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:17:12,255 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:17:12,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:17:12,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:14,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:16,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:18,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:20,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:23,387 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:17:23,387 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:17:26,952 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:17:32,296 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:17:32,297 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:17:32,297 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:17:32,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:17:32,914 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:34,914 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:36,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:38,524 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:17:38,525 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:17:38,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:40,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:52,336 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:17:52,336 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:17:52,337 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:17:52,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:17:53,680 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:17:53,680 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:17:53,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:55,924 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:57,068 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:17:57,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:17:59,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:01,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:08,817 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:18:08,817 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:18:12,373 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:18:12,374 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:18:12,374 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:18:12,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:18:13,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:15,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:17,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:19,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:21,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:23,950 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:18:23,950 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:18:27,146 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:18:32,410 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:18:32,410 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:18:32,411 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:18:32,941 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:18:33,941 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:35,942 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:37,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:39,082 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:18:39,083 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:18:39,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:41,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:52,448 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:18:52,449 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:18:52,449 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:18:52,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:18:53,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:54,223 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:18:54,224 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:18:55,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:57,225 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:18:57,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:18:59,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:01,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:09,379 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:19:09,379 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:19:12,483 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:19:12,484 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:19:12,484 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:19:12,959 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:19:13,959 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:15,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:17,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:19,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:21,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:24,510 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:19:24,511 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:19:27,302 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:19:32,549 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:19:32,549 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:19:32,550 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:19:32,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:19:33,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:35,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:37,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:39,642 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:19:39,643 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:19:39,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:41,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:52,587 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:19:52,587 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:19:52,588 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:19:52,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:19:53,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:54,810 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:19:54,810 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:19:55,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:57,379 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:19:57,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:19:59,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:01,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:09,950 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:20:09,951 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:20:12,664 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:20:12,664 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:20:12,665 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:20:12,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:20:13,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:15,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:17,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:19,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:21,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:25,082 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:20:25,083 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:20:27,457 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:20:32,703 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:20:32,704 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:20:32,704 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:20:32,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:20:33,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:35,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:37,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:39,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:40,215 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:20:40,215 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:20:41,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:52,745 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:20:52,745 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:20:52,746 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:20:53,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:20:54,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:55,351 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:20:55,352 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:20:56,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:20:57,536 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:20:58,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:00,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:02,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:10,483 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:21:10,484 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:21:12,783 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:21:12,783 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:21:12,783 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:21:13,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:21:14,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:16,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:18,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:20,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:22,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:25,614 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:21:25,614 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:21:27,614 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:21:32,821 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:21:32,822 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:21:32,822 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:21:33,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:21:34,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:36,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:38,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:40,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:40,743 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:21:40,743 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:21:42,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:52,861 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:21:52,861 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:21:52,861 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:21:53,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:21:54,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:55,874 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:21:55,875 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:21:56,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:21:57,690 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:21:58,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:00,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:02,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:11,007 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:22:11,008 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:22:12,902 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:22:12,902 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:22:12,902 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:22:13,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:22:14,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:16,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:18,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:20,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:22,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:26,139 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:22:26,139 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:22:27,769 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:22:32,940 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:22:32,941 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:22:32,941 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:22:33,049 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:22:34,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:36,051 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:38,052 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:40,052 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:41,269 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:22:41,269 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:22:42,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:52,979 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:22:52,980 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:22:52,980 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:22:53,058 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:22:54,058 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:56,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:22:56,401 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:22:56,401 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:22:57,847 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:22:58,060 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:00,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:02,062 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:11,531 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:23:11,531 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:23:13,019 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:23:13,019 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:23:13,019 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:23:13,067 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:23:14,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:16,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:18,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:20,070 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:26,664 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:23:26,664 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:23:27,925 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:23:33,056 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:23:33,057 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:23:33,057 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:23:33,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:23:34,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:36,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:38,077 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:40,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:41,798 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:23:41,799 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:23:53,095 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:23:53,095 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:23:53,096 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:23:54,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:23:54,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:56,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:23:56,930 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:23:56,930 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:23:58,004 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:23:58,085 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:00,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:02,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:12,064 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:24:12,065 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:24:13,135 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:24:13,135 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:24:13,136 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:24:14,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:24:14,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:15,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:17,092 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:19,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:21,094 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:23,094 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:27,194 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:24:27,194 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:24:28,082 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:24:33,175 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:24:33,175 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:24:33,176 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:24:34,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:24:34,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:35,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:37,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:39,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:41,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:42,325 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:24:42,325 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:24:43,102 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:53,221 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:24:53,221 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:24:53,222 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:24:54,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:24:54,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:55,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:57,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:24:57,460 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:24:57,461 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:24:58,158 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:24:59,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:01,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:03,110 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:12,595 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:25:12,595 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:25:13,269 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:25:13,270 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:25:13,270 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:25:14,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:25:14,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:15,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:17,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:19,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:21,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:27,728 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:25:27,729 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:25:28,236 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:25:33,318 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:25:33,318 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:25:33,318 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:25:34,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:25:34,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:35,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:37,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:39,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:41,126 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:42,859 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:25:42,860 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:25:43,127 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:53,362 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:25:53,363 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:25:53,363 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:25:54,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:25:54,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:55,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:57,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:25:57,992 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:25:57,993 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:25:58,314 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:25:59,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:01,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:03,135 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:13,129 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:26:13,129 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:26:13,426 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:26:13,427 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:26:13,427 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:26:14,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:26:14,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:15,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:17,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:19,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:21,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:23,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:28,260 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:26:28,260 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:26:28,392 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:26:33,490 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:26:33,490 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:26:33,491 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:26:34,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:26:34,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:35,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:37,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:39,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:41,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:43,152 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:43,392 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:26:43,392 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:26:53,527 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:26:53,527 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:26:53,528 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:26:54,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:26:54,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:55,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:57,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:26:58,468 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:26:58,526 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:26:58,526 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:26:59,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:01,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:03,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:13,565 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:27:13,566 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:27:13,566 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:27:13,674 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:27:13,674 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:27:14,164 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:27:15,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:17,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:19,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:21,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:23,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:28,546 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:27:28,813 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:27:28,814 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:27:33,606 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:27:33,606 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:27:33,607 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:27:34,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:27:35,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:37,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:39,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:41,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:43,945 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:27:43,945 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:27:53,644 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:27:53,645 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:27:53,645 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:27:54,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:27:55,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:57,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:27:58,624 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:27:59,090 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:27:59,091 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:27:59,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:01,184 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:13,683 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:28:13,683 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:28:13,684 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:28:14,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:28:14,231 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:28:14,231 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:28:15,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:17,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:19,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:21,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:23,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:28,702 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:28:29,386 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:28:29,386 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:28:33,722 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:28:33,722 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:28:33,722 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:28:34,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:28:35,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:37,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:39,200 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:41,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:43,202 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:44,515 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:28:44,516 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:28:53,758 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:28:53,758 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:28:53,760 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:28:54,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:28:55,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:57,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:58,780 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:28:59,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:28:59,648 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:28:59,648 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:29:01,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:03,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:13,797 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:29:13,798 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:29:13,798 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:29:14,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:29:14,779 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:29:14,779 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:29:15,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:17,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:19,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:21,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:23,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:28,858 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:29:29,939 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:29:29,940 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:29:33,836 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:29:33,837 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:29:33,837 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:29:34,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:29:35,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:37,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:39,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:41,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:43,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:45,071 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:29:45,072 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:29:53,875 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:29:53,875 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:29:53,876 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:29:54,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:29:55,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:57,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:29:58,937 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:29:59,234 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:00,201 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:30:00,201 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:30:01,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:03,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:13,910 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:30:13,910 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:30:13,911 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:30:14,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:30:15,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:15,358 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:30:15,359 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:30:17,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:19,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:21,243 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:23,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:29,016 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:30:30,492 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:30:30,492 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:30:33,948 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:30:33,948 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:30:33,949 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:30:34,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:30:35,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:37,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:39,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:41,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:45,627 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:30:45,627 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:30:53,985 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:30:53,985 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:30:53,985 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:30:54,258 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:30:55,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:56,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:58,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:30:59,095 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:31:00,261 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:00,762 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:31:00,762 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:31:02,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:14,022 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:31:14,022 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:31:14,023 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:31:14,267 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:31:15,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:15,908 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:31:15,909 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:31:16,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:18,269 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:20,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:22,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:29,173 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:31:31,043 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:31:31,043 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:31:34,062 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:31:34,063 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:31:34,063 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:31:34,276 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:31:35,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:36,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:38,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:40,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:42,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:46,176 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:31:46,177 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:31:54,102 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:31:54,102 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:31:54,103 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:31:54,284 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:31:55,284 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:56,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:58,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:31:59,251 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:32:00,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:01,308 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:32:01,309 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:32:02,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:14,139 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:32:14,140 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:32:14,140 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:32:14,292 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:32:15,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:16,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:16,449 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:32:16,449 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:32:18,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:20,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:22,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:29,331 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:32:31,586 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:32:31,587 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:32:34,180 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:32:34,180 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:32:34,180 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:32:34,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:32:35,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:36,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:38,302 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:40,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:42,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:46,715 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:32:46,716 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:32:54,218 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:32:54,218 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:32:54,218 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:32:54,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:32:55,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:56,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:58,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:32:59,409 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:33:00,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:01,848 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:33:01,848 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:33:02,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:14,259 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:33:14,259 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:33:14,260 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:33:14,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:33:15,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:16,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:17,002 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:33:17,003 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:33:18,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:20,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:22,319 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:29,489 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:33:32,160 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:33:32,160 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:33:34,302 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:33:34,303 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:33:34,303 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:33:34,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:33:35,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:36,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:38,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:40,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:42,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:47,289 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:33:47,289 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:33:54,339 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:33:54,340 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:33:54,340 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:33:55,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:33:55,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:56,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:58,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:33:59,568 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:34:00,335 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:34:02,336 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:34:02,421 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:34:02,422 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:34:14,379 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:34:14,380 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:34:14,380 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:34:15,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:34:15,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:34:16,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:34:17,915 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:34:17,916 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:34:23,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:34:25,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:34:27,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:34:29,647 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:34:33,234 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:34:33,235 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status