diff --git "a/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" "b/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" --- "a/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" +++ "b/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" @@ -8321,3 +8321,7432 @@ 2021-07-15 06:41:18,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log 2021-07-15 06:41:29,275 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status 2021-07-15 06:41:29,275 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:41:39,958 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:41:44,414 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:41:44,508 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:41:59,640 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:41:59,641 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:42:10,033 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:42:14,777 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:42:14,777 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:42:28,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:29,948 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:42:29,948 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:42:30,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:32,770 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:34,771 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:36,772 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:40,107 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:42:45,081 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:42:45,081 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:42:46,616 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:42:46,617 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:42:46,617 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:42:46,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:42:47,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:48,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:50,778 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:52,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:42:54,780 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:00,212 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:43:00,212 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:43:06,741 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:43:06,741 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:43:06,742 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:43:06,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:43:07,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:08,787 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:10,206 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:43:10,788 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:12,788 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:14,789 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:15,344 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:43:15,345 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:43:26,896 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:43:26,897 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:43:26,897 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:43:27,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:43:27,796 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:28,796 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:30,665 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:43:30,665 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:43:30,797 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:32,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:34,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:36,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:40,281 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:43:45,816 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:43:45,816 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:43:46,989 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:43:46,990 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:43:46,990 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:43:47,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:43:47,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:48,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:50,805 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:53,807 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:55,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:43:57,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:00,948 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:44:00,948 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:44:07,155 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:44:07,156 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:44:07,156 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:44:07,813 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:44:07,813 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:09,814 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:10,357 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:44:11,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:13,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:15,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:16,116 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:44:16,116 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:44:27,286 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:44:27,287 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:44:27,287 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:44:27,822 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:44:28,823 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:29,823 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:31,295 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:44:31,296 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:44:31,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:33,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:35,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:37,827 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:40,432 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:44:46,450 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:44:46,450 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:44:47,440 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:44:47,440 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:44:47,441 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:44:47,832 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:44:48,832 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:49,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:51,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:53,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:44:55,835 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:01,593 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:45:01,593 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:45:07,495 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:45:07,496 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:45:07,496 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:45:07,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:45:08,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:09,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:10,510 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:45:11,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:13,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:15,844 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:16,727 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:45:16,727 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:45:27,622 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:45:27,623 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:45:27,623 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:45:27,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:45:28,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:29,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:31,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:31,895 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:45:31,896 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:45:33,852 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:35,853 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:40,583 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:45:47,090 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:45:47,091 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:45:47,679 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:45:47,680 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:45:47,680 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:45:47,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:45:48,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:49,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:51,860 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:54,861 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:45:56,862 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:02,220 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:46:02,220 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:46:07,736 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:46:07,737 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:46:07,737 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:46:07,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:46:08,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:10,658 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:46:10,869 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:12,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:14,871 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:16,872 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:17,351 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:46:17,352 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:46:27,902 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:46:27,903 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:46:27,903 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:46:28,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:46:28,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:30,878 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:32,498 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:46:32,499 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:46:32,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:34,880 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:36,881 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:40,735 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:46:47,674 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:46:47,675 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:46:47,978 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:46:47,978 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:46:47,978 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:46:48,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:46:48,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:49,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:50,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:52,888 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:54,889 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:46:56,890 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:02,807 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:47:02,807 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:47:08,172 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:47:08,173 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:47:08,173 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:47:08,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:47:08,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:09,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:10,812 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:47:10,896 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:12,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:14,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:16,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:17,939 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:47:17,939 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:47:28,262 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:47:28,262 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:47:28,263 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:47:28,904 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:47:28,904 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:29,905 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:30,905 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:32,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:33,075 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:47:33,075 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:47:35,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:37,909 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:40,893 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:47:48,211 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:47:48,212 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:47:48,605 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:47:48,605 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:47:48,605 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:47:48,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:47:49,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:51,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:53,917 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:55,918 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:47:57,919 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:03,345 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:48:03,345 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:48:08,732 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:48:08,733 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:48:08,733 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:48:08,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:48:09,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:10,974 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:48:11,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:13,926 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:15,927 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:17,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:18,478 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:48:18,478 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:48:28,947 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:48:28,947 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:48:28,948 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:48:29,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:48:29,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:31,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:33,627 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:48:33,627 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:48:33,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:35,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:37,938 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:41,056 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:48:48,787 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:48:48,788 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:48:49,135 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:48:49,136 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:48:49,136 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:48:49,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:48:49,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:51,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:53,945 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:56,947 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:48:58,948 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:03,918 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:49:03,919 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:49:09,336 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:49:09,336 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:49:09,337 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:49:09,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:49:09,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:10,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:11,132 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:49:12,955 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:14,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:16,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:18,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:19,254 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:49:19,254 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:49:29,458 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:49:29,459 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:49:29,459 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:49:29,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:49:29,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:30,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:32,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:34,402 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:49:34,402 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:49:34,966 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:36,967 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:38,968 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:41,212 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:49:49,553 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:49:49,553 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:49:49,685 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:49:49,685 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:49:49,685 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:49:49,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:49:50,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:52,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:54,975 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:56,976 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:49:58,977 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:04,704 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:50:04,705 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:50:09,763 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:50:09,763 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:50:09,763 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:50:09,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:50:10,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:11,289 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:50:12,983 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:14,984 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:16,985 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:18,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:19,838 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:50:19,839 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:50:29,852 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:50:29,852 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:50:29,853 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:50:29,990 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:50:30,991 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:33,992 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:35,050 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:50:35,050 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:50:35,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:37,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:39,995 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:41,366 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:50:49,901 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:50:49,902 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:50:49,902 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:50:49,999 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:50:50,230 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:50:50,230 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:50:51,000 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:52,000 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:54,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:56,002 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:50:58,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:05,416 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:51:05,416 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:51:09,968 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:51:09,969 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:51:09,969 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:51:10,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:51:11,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:11,444 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:51:12,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:14,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:16,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:18,012 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:20,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:20,545 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:51:20,545 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:51:30,087 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:51:30,088 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:51:30,088 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:51:31,018 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:51:31,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:32,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:34,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:35,725 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:51:35,725 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:51:36,021 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:38,022 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:41,523 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:51:50,227 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:51:50,228 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:51:50,228 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:51:50,901 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:51:50,901 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:51:51,028 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:51:52,028 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:53,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:54,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:56,030 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:51:58,031 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:00,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:06,079 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:52:06,080 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:52:10,322 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:52:10,322 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:52:10,322 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:52:11,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:52:11,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:11,605 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:52:12,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:15,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:17,040 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:19,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:21,212 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:52:21,213 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:52:30,510 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:52:30,511 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:52:30,511 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:52:31,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:52:33,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:35,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:36,414 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:52:36,414 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:52:37,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:39,050 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:41,051 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:41,684 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:52:50,657 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:52:50,657 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:52:50,657 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:52:51,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:52:51,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:51,663 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:52:51,663 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:52:53,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:55,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:57,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:52:59,059 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:06,797 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:53:06,797 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:53:10,743 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:53:10,743 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:53:10,744 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:53:11,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:53:11,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:11,762 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:53:13,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:15,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:17,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:19,067 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:21,068 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:21,951 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:53:21,952 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:53:30,903 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:53:30,904 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:53:30,904 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:53:31,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:53:33,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:34,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:35,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:37,075 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:37,743 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:53:37,743 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:53:39,076 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:41,077 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:41,843 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:53:51,053 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:53:51,054 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:53:51,054 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:53:51,081 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:53:52,904 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:53:52,904 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:53:53,082 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:54,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:56,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:53:58,084 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:00,085 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:08,040 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:54:08,040 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:54:11,128 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:54:11,129 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:54:11,129 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:54:11,921 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:54:12,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:54:12,091 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:14,091 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:16,092 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:18,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:20,094 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:23,171 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:54:23,172 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:54:31,199 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:54:31,199 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:54:31,200 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:54:32,100 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:54:34,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:36,102 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:38,103 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:38,306 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:54:38,306 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:54:40,104 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:42,003 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:54:51,272 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:54:51,273 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:54:51,273 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:54:52,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:54:52,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:53,486 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:54:53,486 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:54:54,110 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:56,111 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:54:58,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:00,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:08,698 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:55:08,698 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:55:11,448 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:55:11,449 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:55:11,449 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:55:12,085 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:55:12,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:55:12,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:14,118 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:16,119 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:18,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:20,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:23,833 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:55:23,833 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:55:31,540 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:55:31,540 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:55:31,540 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:55:32,126 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:55:32,126 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:34,127 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:36,128 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:38,129 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:39,049 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:55:39,050 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:55:40,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:42,164 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:55:51,670 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:55:51,670 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:55:51,671 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:55:52,135 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:55:52,135 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:54,136 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:54,253 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:55:54,253 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:55:57,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:55:59,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:01,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:09,393 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:56:09,394 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:56:11,813 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:56:11,813 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:56:11,814 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:56:12,143 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:56:12,143 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:12,242 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:56:13,144 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:15,145 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:17,146 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:19,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:21,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:24,562 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:56:24,563 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:56:31,945 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:56:31,945 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:56:31,946 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:56:32,152 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:56:33,153 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:35,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:37,155 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:39,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:39,700 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:56:39,700 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:56:41,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:42,324 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:56:52,090 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:56:52,091 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:56:52,091 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:56:52,161 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:56:53,162 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:54,879 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:56:54,879 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:56:55,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:57,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:56:59,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:01,166 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:10,059 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:57:10,059 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:57:12,234 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:57:12,234 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:57:12,235 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:57:12,405 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:57:13,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:57:13,171 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:15,171 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:18,173 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:20,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:22,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:25,188 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:57:25,189 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:57:32,417 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:57:32,417 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:57:32,417 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:57:33,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:57:34,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:35,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:36,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:38,181 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:40,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:40,323 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:57:40,323 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:57:42,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:42,476 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:57:52,526 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:57:52,526 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:57:52,527 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:57:53,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:57:54,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:55,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:55,526 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:57:55,526 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:57:56,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:57:58,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:00,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:02,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:10,717 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:58:10,718 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:58:12,543 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:58:12,666 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:58:12,666 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:58:12,667 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:58:13,196 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:58:14,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:15,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:16,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:18,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:20,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:22,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:25,850 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:58:25,851 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:58:32,791 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:58:32,791 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:58:32,792 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:58:33,204 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:58:34,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:35,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:36,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:38,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:40,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:40,981 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:58:40,982 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:58:42,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:42,613 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:58:52,941 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:58:52,942 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:58:52,942 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:58:53,213 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:58:54,213 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:55,213 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:56,131 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:58:56,131 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:58:56,214 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:58:58,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:00,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:03,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:11,281 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:59:11,282 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:59:12,680 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:59:13,063 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:59:13,063 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:59:13,064 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:59:13,222 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:59:15,223 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:17,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:19,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:21,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:23,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:26,413 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:59:26,413 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:59:33,200 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:59:33,201 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:59:33,201 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:59:33,230 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:59:35,231 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:36,231 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:37,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:39,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:41,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:41,548 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:59:41,548 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:59:42,747 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 06:59:43,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:53,328 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 06:59:53,328 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 06:59:53,328 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 06:59:54,239 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 06:59:54,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:55,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:56,751 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 06:59:56,752 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 06:59:57,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 06:59:59,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:01,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:11,884 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:00:11,885 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:00:12,817 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:00:13,453 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:00:13,453 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:00:13,454 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:00:14,248 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:00:15,248 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:16,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:17,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:19,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:21,251 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:23,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:27,025 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:00:27,025 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:00:33,582 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:00:33,582 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:00:33,583 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:00:34,256 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:00:35,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:36,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:37,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:39,259 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:41,259 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:42,157 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:00:42,157 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:00:42,889 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:00:53,672 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:00:53,672 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:00:53,673 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:00:54,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:00:55,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:56,266 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:57,266 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:00:57,365 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:00:57,366 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:00:59,267 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:01,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:12,556 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:01:12,556 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:01:12,958 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:01:13,818 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:01:13,818 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:01:13,819 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:01:14,274 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:01:15,274 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:16,275 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:18,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:20,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:22,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:27,688 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:01:27,688 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:01:33,959 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:01:33,959 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:01:33,960 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:01:34,282 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:01:36,283 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:38,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:40,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:42,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:43,031 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:01:43,602 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:01:43,602 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:01:44,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:54,081 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:01:54,081 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:01:54,082 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:01:54,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:01:56,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:58,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:01:58,804 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:01:58,804 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:02:00,295 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:02,296 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:04,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:13,101 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:02:13,990 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:02:13,990 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:02:14,280 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:02:14,284 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:02:14,284 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:02:14,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:02:16,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:18,303 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:20,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:22,305 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:29,122 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:02:29,122 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:02:34,352 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:02:34,352 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:02:34,352 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:02:35,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:02:36,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:37,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:38,313 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:40,314 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:42,314 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:43,174 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:02:44,254 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:02:44,254 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:02:54,458 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:02:54,459 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:02:54,459 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:02:55,321 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:02:56,321 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:57,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:58,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:02:59,392 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:02:59,392 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:03:00,323 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:02,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:13,246 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:03:14,573 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:03:14,574 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:03:14,704 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:03:14,704 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:03:14,704 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:03:15,330 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:03:16,331 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:17,331 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:19,332 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:21,334 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:23,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:29,723 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:03:29,723 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:03:34,669 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:03:34,669 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:03:34,670 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:03:35,340 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:03:37,341 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:39,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:41,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:43,318 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:03:43,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:44,854 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:03:44,854 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:03:55,025 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:03:55,025 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:03:55,025 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:03:55,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:03:57,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:59,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:03:59,994 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:03:59,994 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:04:01,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:03,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:13,389 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:04:15,109 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:04:15,110 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:04:15,111 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:04:15,197 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:04:15,197 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:04:15,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:04:17,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:19,361 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:21,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:23,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:30,371 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:04:30,372 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:04:35,199 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:04:35,199 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:04:35,200 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:04:35,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:04:37,370 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:39,371 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:41,372 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:43,373 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:43,463 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:04:45,374 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:45,511 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:04:45,511 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:04:55,253 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:04:55,253 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:04:55,253 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:04:55,378 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:04:57,379 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:04:59,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:00,715 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:05:00,716 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:05:01,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:03,381 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:13,535 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:05:15,355 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:05:15,355 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:05:15,356 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:05:15,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:05:15,945 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:05:15,945 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:05:17,388 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:19,389 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:21,390 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:23,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:31,199 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:05:31,199 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:05:35,433 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:05:35,434 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:05:35,434 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:05:36,396 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:05:37,397 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:39,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:41,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:43,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:43,611 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:05:46,334 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:05:46,334 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:05:55,562 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:05:55,562 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:05:55,563 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:05:56,405 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:05:57,405 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:05:59,406 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:01,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:01,465 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:06:01,466 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:06:03,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:13,690 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:06:15,671 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:06:15,672 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:06:15,672 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:06:16,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:06:16,608 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:06:16,608 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:06:17,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:20,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:22,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:24,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:31,751 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:06:31,751 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:06:35,915 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:06:35,916 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:06:35,916 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:06:36,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:06:38,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:40,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:42,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:43,764 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:06:44,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:06:46,886 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:06:46,886 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:06:55,979 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:06:55,979 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:06:55,980 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:06:56,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:06:58,435 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:00,436 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:02,074 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:07:02,075 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:07:02,437 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:04,438 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:13,846 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:07:16,092 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:07:16,092 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:07:16,093 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:07:16,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:07:17,263 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:07:17,263 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:07:18,444 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:20,445 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:22,446 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:24,447 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:32,408 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:07:32,408 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:07:36,291 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:07:36,292 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:07:36,292 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:07:36,452 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:07:38,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:40,454 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:42,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:43,924 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:07:44,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:07:47,538 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:07:47,539 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:07:56,417 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:07:56,418 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:07:56,418 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:07:56,462 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:07:58,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:00,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:02,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:02,715 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:08:02,715 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:08:04,466 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:14,003 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:08:16,487 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:08:16,487 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:08:16,513 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:08:17,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:08:17,911 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:08:17,912 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:08:18,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:20,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:22,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:24,474 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:33,060 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:08:33,061 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:08:36,605 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:08:36,606 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:08:36,606 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:08:37,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:08:38,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:41,482 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:43,483 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:44,084 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:08:45,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:48,201 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:08:48,201 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:08:56,766 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:08:56,767 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:08:56,767 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:08:57,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:08:58,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:08:59,490 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:01,491 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:03,360 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:09:03,361 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:09:03,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:05,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:07,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:14,164 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:09:16,939 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:09:16,939 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:09:16,940 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:09:17,498 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:09:18,498 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:18,541 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:09:18,542 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:09:19,499 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:21,500 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:23,500 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:25,501 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:27,503 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:33,688 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:09:33,689 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:09:37,085 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:09:37,086 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:09:37,086 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:09:37,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:09:38,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:39,508 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:41,509 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:43,510 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:44,244 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:09:45,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:47,512 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:48,821 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:09:48,821 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:09:57,185 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:09:57,186 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:09:57,186 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:09:57,516 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:09:58,516 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:09:59,517 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:01,518 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:03,519 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:04,004 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:10:04,004 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:10:05,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:07,521 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:14,322 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:10:17,371 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:10:17,371 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:10:17,372 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:10:17,525 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:10:18,525 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:19,215 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:10:19,216 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:10:19,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:21,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:23,527 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:25,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:34,376 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:10:34,377 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:10:37,430 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:10:37,430 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:10:37,431 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:10:37,533 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:10:39,534 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:41,535 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:43,536 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:44,401 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:10:45,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:10:49,508 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:10:49,508 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:10:57,545 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:10:57,545 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:10:57,545 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:10:58,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:10:59,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:01,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:03,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:04,655 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:11:04,656 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:11:05,545 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:14,482 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:11:17,665 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:11:17,666 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:11:17,666 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:11:18,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:11:19,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:19,908 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:11:19,908 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:11:21,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:24,553 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:26,554 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:28,555 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:35,071 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:11:35,072 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:11:37,762 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:11:37,763 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:11:37,763 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:11:38,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:11:39,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:40,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:42,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:44,560 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:11:44,564 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:46,565 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:11:50,208 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:11:50,208 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:11:57,840 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:11:57,840 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:11:57,841 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:11:58,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:11:59,571 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:00,571 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:02,572 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:04,573 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:05,594 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:12:05,595 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:12:06,574 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:14,639 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:12:17,962 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:12:17,963 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:12:17,963 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:12:18,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:12:19,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:20,580 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:20,745 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:12:20,745 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:12:22,581 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:24,581 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:26,582 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:28,583 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:35,880 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:12:35,881 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:12:38,150 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:12:38,151 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:12:38,151 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:12:38,588 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:12:39,588 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:40,589 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:42,590 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:44,591 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:44,721 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:12:46,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:12:51,012 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:12:51,012 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:12:58,304 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:12:58,305 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:12:58,305 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:12:58,597 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:12:59,597 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:00,598 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:02,599 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:04,600 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:06,191 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:13:06,191 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:13:06,601 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:08,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:14,802 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:13:18,441 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:13:18,441 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:13:18,442 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:13:18,606 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:13:19,607 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:20,607 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:21,439 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:13:21,439 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:13:22,608 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:24,609 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:26,609 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:36,623 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:13:36,623 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:13:38,554 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:13:38,555 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:13:38,555 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:13:38,614 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:13:40,615 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:42,616 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:44,885 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:13:45,617 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:47,618 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:13:51,755 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:13:51,755 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:13:58,639 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:13:58,639 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:13:58,640 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:13:59,623 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:14:00,624 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:01,624 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:03,625 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:05,626 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:07,168 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:14:07,168 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:14:07,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:14,965 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:14:18,701 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:14:18,701 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:14:18,702 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:14:19,632 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:14:20,632 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:21,633 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:22,313 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:14:22,313 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:14:23,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:25,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:27,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:37,447 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:14:37,447 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:14:38,820 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:14:38,820 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:14:38,821 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:14:39,640 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:14:39,640 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:40,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:41,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:43,642 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:45,048 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:14:45,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:47,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:14:52,599 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:14:52,599 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:14:58,967 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:14:58,968 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:14:58,968 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:14:59,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:15:00,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:01,650 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:03,651 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:05,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:07,653 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:07,731 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:15:07,731 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:15:15,127 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:15:19,045 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:15:19,046 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:15:19,047 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:15:19,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:15:20,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:21,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:22,908 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:15:22,908 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:15:23,659 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:25,660 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:27,661 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:38,040 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:15:38,040 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:15:39,176 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:15:39,176 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:15:39,177 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:15:39,667 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:15:40,668 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:41,668 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:43,669 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:45,207 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:15:45,670 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:47,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:49,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:15:53,171 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:15:53,171 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:15:59,287 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:15:59,288 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:15:59,288 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:15:59,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:16:00,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:01,678 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:04,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:06,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:08,301 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:16:08,301 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:16:08,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:15,285 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:16:19,366 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:16:19,367 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:16:19,368 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:16:19,687 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:16:20,688 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:22,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:23,444 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:16:23,444 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:16:24,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:26,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:28,691 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:38,593 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:16:38,594 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:16:39,485 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:16:39,485 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:16:39,486 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:16:39,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:16:40,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:41,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:42,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:44,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:45,363 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:16:46,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:48,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:16:53,726 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:16:53,726 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:16:59,644 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:16:59,645 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:16:59,645 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:16:59,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:17:01,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:02,708 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:04,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:06,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:08,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:08,858 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:17:08,858 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:17:15,445 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:17:19,741 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:17:19,741 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:17:19,741 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:17:20,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:17:20,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:21,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:22,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:24,003 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:17:24,004 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:17:24,716 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:26,717 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:28,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:39,136 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:17:39,137 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:17:39,791 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:17:39,791 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:17:39,792 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:17:40,723 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:17:40,723 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:41,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:42,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:44,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:45,525 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:17:46,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:48,727 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:17:54,268 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:17:54,269 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:17:59,838 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:17:59,838 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:17:59,839 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:18:00,731 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:18:00,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:01,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:02,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:04,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:06,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:08,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:09,401 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:18:09,401 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:18:15,607 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:18:19,922 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:18:19,922 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:18:19,923 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:18:20,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:18:20,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:21,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:23,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:24,609 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:18:24,609 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:18:25,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:27,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:29,744 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:39,811 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:18:39,811 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:18:40,031 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:18:40,032 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:18:40,032 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:18:40,748 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:18:41,749 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:43,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:45,687 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:18:45,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:47,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:49,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:18:54,943 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:18:54,943 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:19:00,122 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:19:00,122 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:19:00,123 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:19:00,757 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:19:01,757 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:03,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:05,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:07,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:09,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:10,078 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:19:10,078 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:19:15,755 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:19:20,220 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:19:20,221 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:19:20,221 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:19:20,765 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:19:21,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:23,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:25,271 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:19:25,271 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:19:25,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:27,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:40,393 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:19:40,394 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:19:40,394 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:19:40,469 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:19:40,469 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:19:40,773 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:19:42,774 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:44,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:45,824 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:19:46,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:48,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:19:55,751 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:19:55,751 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:20:00,728 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:20:00,729 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:20:00,729 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:20:00,782 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:20:02,782 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:04,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:06,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:08,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:10,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:10,885 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:20:10,885 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:20:15,896 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:20:20,798 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:20:20,799 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:20:20,799 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:20:21,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:20:22,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:24,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:26,096 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:20:26,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:20:26,793 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:28,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:40,983 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:20:40,984 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:20:40,984 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:20:41,344 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:20:41,344 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:20:41,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:20:42,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:44,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:45,967 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:20:46,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:48,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:20:56,485 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:20:56,485 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:21:01,083 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:21:01,083 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:21:01,084 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:21:01,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:21:02,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:04,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:06,810 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:08,811 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:11,616 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:21:11,616 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:21:16,039 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:21:21,181 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:21:21,181 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:21:21,182 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:21:21,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:21:22,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:24,818 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:26,753 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:21:26,753 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:21:26,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:28,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:30,820 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:41,309 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:21:41,310 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:21:41,311 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:21:41,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:21:41,958 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:21:41,958 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:21:42,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:43,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:45,827 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:46,112 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:21:47,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:49,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:21:57,149 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:21:57,149 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:22:01,373 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:22:01,374 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:22:01,374 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:22:01,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:22:02,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:03,835 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:05,836 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:07,837 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:09,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:12,283 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:22:12,283 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:22:16,187 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:22:21,461 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:22:21,461 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:22:21,462 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:22:21,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:22:22,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:23,844 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:25,845 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:27,455 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:22:27,455 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:22:27,846 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:29,847 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:41,678 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:22:41,679 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:22:41,679 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:22:41,852 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:22:42,657 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:22:42,658 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:22:42,853 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:43,853 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:45,854 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:46,276 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:22:47,855 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:49,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:22:57,790 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:22:57,790 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:23:01,794 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:23:01,794 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:23:01,794 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:23:01,862 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:23:02,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:03,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:05,864 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:07,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:09,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:12,927 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:23:12,927 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:23:16,352 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:23:21,892 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:23:21,892 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:23:21,892 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:23:22,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:23:23,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:25,874 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:27,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:28,204 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:23:28,205 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:23:29,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:42,006 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:23:42,007 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:23:42,007 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:23:42,882 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:23:43,437 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:23:43,437 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:23:43,883 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:44,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:46,431 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:23:46,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:48,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:50,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:23:58,574 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:23:58,575 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:24:02,152 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:24:02,152 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:24:02,152 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:24:02,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:24:03,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:04,894 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:06,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:08,896 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:10,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:12,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:13,705 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:24:13,706 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:24:16,507 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:24:22,321 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:24:22,322 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:24:22,322 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:24:22,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:24:23,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:24,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:26,904 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:28,905 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:24:28,906 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:24:28,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:30,907 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:42,401 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:24:42,401 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:24:42,402 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:24:42,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:24:43,912 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:44,085 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:24:44,085 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:24:44,912 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:46,585 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:24:46,913 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:48,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:50,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:24:59,227 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:24:59,228 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:25:02,508 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:25:02,508 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:25:02,509 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:25:02,919 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:25:03,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:04,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:06,921 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:08,922 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:10,923 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:14,359 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:25:14,360 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:25:16,660 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:25:22,634 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:25:22,634 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:25:22,635 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:25:22,927 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:25:23,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:24,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:26,929 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:28,930 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:29,504 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:25:29,505 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:25:30,930 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:32,931 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:42,807 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:25:42,807 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:25:42,808 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:25:42,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:25:43,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:44,675 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:25:44,675 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:25:44,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:46,734 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:25:46,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:48,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:50,938 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:25:59,819 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:25:59,819 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:26:02,981 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:26:02,982 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:26:02,982 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:26:03,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:26:03,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:04,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:07,945 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:09,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:11,947 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:14,951 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:26:14,952 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:26:16,812 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:26:23,108 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:26:23,109 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:26:23,110 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:26:23,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:26:24,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:25,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:27,955 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:29,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:30,133 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:26:30,133 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:26:31,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:43,205 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:26:43,205 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:26:43,206 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:26:43,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:26:44,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:45,342 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:26:45,342 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:26:45,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:46,888 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:26:47,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:49,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:26:51,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:00,501 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:27:00,502 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:27:03,282 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:27:03,282 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:27:03,283 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:27:03,970 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:27:04,971 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:05,971 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:07,972 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:09,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:11,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:15,637 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:27:15,637 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:27:16,963 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:27:23,335 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:27:23,335 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:27:23,336 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:27:23,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:27:24,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:25,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:27,981 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:29,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:30,820 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:27:30,820 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:27:31,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:43,399 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:27:43,399 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:27:43,400 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:27:43,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:27:43,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:44,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:45,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:46,055 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:27:46,055 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:27:47,037 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:27:47,990 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:49,991 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:27:51,992 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:01,190 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:28:01,190 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:28:03,514 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:28:03,514 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:28:03,514 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:28:03,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:28:03,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:04,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:05,998 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:07,999 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:09,999 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:12,000 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:16,324 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:28:16,325 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:28:17,116 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:28:23,641 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:28:23,641 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:28:23,642 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:28:24,005 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:28:25,005 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:26,006 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:28,007 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:31,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:31,461 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:28:31,461 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:28:33,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:43,725 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:28:43,726 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:28:43,726 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:28:44,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:28:45,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:46,597 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:28:46,598 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:28:47,014 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:47,197 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:28:49,015 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:51,016 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:28:53,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:01,801 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:29:01,801 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:29:03,824 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:29:03,825 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:29:03,825 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:29:04,021 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:29:05,022 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:07,022 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:09,023 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:11,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:13,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:16,932 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:29:16,933 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:29:17,277 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:29:23,913 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:29:23,914 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:29:23,914 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:29:24,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:29:25,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:27,030 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:29,031 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:31,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:32,065 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:29:32,066 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:29:33,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:44,048 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:29:44,048 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:29:44,048 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:29:45,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:29:45,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:46,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:47,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:47,274 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:29:47,274 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:29:47,402 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:29:49,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:51,040 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:29:53,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:02,428 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:30:02,429 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:30:04,211 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:30:04,211 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:30:04,212 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:30:05,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:30:06,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:07,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:09,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:11,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:13,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:17,441 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:30:17,560 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:30:17,561 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:30:24,295 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:30:24,296 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:30:24,296 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:30:25,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:30:25,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:26,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:27,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:29,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:31,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:32,715 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:30:32,715 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:30:34,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:44,486 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:30:44,486 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:30:44,487 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:30:45,062 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:30:46,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:47,521 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:30:47,894 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:30:47,895 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:30:48,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:50,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:52,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:30:54,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:03,095 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:31:03,096 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:31:04,677 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:31:04,678 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:31:04,678 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:31:05,070 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:31:06,071 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:08,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:10,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:12,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:14,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:17,603 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:31:18,239 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:31:18,239 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:31:24,829 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:31:24,829 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:31:24,830 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:31:25,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:31:26,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:28,080 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:30,081 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:32,082 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:33,407 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:31:33,407 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:31:34,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:44,905 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:31:44,905 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:31:44,906 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:31:45,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:31:46,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:47,686 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:31:48,088 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:48,600 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:31:48,600 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:31:50,089 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:31:52,089 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:03,748 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:32:03,748 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:32:04,983 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:32:04,984 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:32:04,984 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:32:05,095 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:32:06,095 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:08,096 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:10,097 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:13,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:15,099 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:17,768 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:32:18,884 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:32:18,884 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:32:25,056 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:32:25,056 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:32:25,057 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:32:25,103 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:32:26,104 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:27,104 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:29,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:31,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:33,107 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:34,017 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:32:34,017 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:32:35,108 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:45,206 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:32:45,207 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:32:45,207 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:32:46,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:32:47,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:47,846 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:32:49,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:49,147 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:32:49,147 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:32:51,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:32:53,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:04,299 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:33:04,300 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:33:05,373 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:33:05,374 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:33:05,374 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:33:06,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:33:07,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:09,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:11,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:13,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:17,928 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:33:19,432 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:33:19,432 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:33:25,431 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:33:25,432 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:33:25,432 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:33:26,129 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:33:27,129 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:29,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:31,131 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:33,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:34,564 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:33:34,564 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:33:35,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:45,509 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:33:45,509 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:33:45,509 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:33:46,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:33:47,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:48,006 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:33:49,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:49,762 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:33:49,763 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:33:51,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:53,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:33:55,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:04,900 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:34:04,901 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:34:05,620 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:34:05,621 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:34:05,621 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:34:06,145 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:34:07,146 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:09,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:11,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:13,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:18,084 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:34:20,033 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:34:20,034 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:34:25,750 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:34:25,750 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:34:25,750 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:34:26,153 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:34:27,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:28,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:30,155 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:32,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:34,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:35,170 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:34:35,170 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:34:45,896 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:34:45,896 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:34:45,897 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:34:46,162 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:34:47,162 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:48,161 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:34:48,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:50,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:50,327 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:34:50,327 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:34:52,166 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:34:54,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:05,513 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:35:05,514 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:35:06,008 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:35:06,009 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:35:06,009 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:35:06,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:35:07,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:08,173 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:10,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:12,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:14,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:16,176 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:18,241 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:35:20,648 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:35:20,648 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:35:26,078 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:35:26,078 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:35:26,079 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:35:26,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:35:27,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:28,181 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:30,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:32,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:34,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:35,780 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:35:35,780 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:35:46,155 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:35:46,156 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:35:46,156 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:35:46,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:35:47,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:48,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:48,322 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:35:50,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:50,929 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:35:50,929 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:35:52,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:54,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:35:56,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:06,064 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:36:06,064 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:36:06,457 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:36:06,457 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:36:06,458 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:36:07,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:36:08,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:10,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:12,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:14,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:16,201 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:18,391 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:36:21,198 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:36:21,198 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:36:26,608 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:36:26,608 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:36:26,609 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:36:27,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:36:28,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:29,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:31,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:33,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:35,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:36,329 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:36:36,330 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:36:46,716 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:36:46,716 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:36:46,717 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:36:47,214 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:36:48,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:48,461 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:36:49,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:51,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:51,510 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:36:51,511 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:36:53,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:55,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:36:57,219 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:06,641 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:37:06,642 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:37:06,912 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:37:06,913 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:37:06,913 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:37:07,223 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:37:08,223 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:09,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:11,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:13,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:15,227 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:18,535 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:37:21,775 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:37:21,775 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:37:27,133 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:37:27,134 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:37:27,134 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:37:27,231 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:37:28,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:29,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:31,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:33,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:35,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:36,927 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:37:36,927 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:37:47,311 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:37:47,312 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:37:47,312 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:37:48,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:37:48,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:48,610 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:37:49,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:51,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:52,122 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:37:52,122 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:37:53,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:37:55,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:07,266 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:38:07,267 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:38:07,396 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:38:07,397 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:38:07,397 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:38:08,248 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:38:08,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:09,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:11,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:13,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:16,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:18,685 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:38:22,428 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:38:22,429 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:38:27,501 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:38:27,501 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:38:27,502 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:38:28,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:38:28,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:30,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:32,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:34,259 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:36,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:37,560 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:38:37,560 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:38:47,658 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:38:47,659 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:38:47,659 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:38:48,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:38:48,768 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:38:49,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:50,266 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:52,267 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:52,733 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:38:52,734 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:38:54,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:38:56,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:07,807 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:39:07,807 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:39:07,808 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:39:07,905 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:39:07,906 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:39:08,273 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:39:09,273 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:10,274 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:12,274 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:14,275 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:16,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:18,847 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:39:23,037 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:39:23,038 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:39:27,882 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:39:27,883 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:39:27,883 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:39:28,281 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:39:29,281 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:30,282 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:32,283 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:34,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:36,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:38,169 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:39:38,169 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:39:47,998 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:39:47,999 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:39:47,999 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:39:48,289 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:39:48,928 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:39:49,290 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:50,290 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:52,291 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:53,322 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:39:53,322 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:39:54,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:39:56,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:08,120 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:40:08,121 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:40:08,121 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:40:08,298 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:40:08,495 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:40:08,495 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:40:09,298 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:10,298 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:12,299 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:14,300 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:16,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:19,007 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:40:23,633 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:40:23,638 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:40:28,211 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:40:28,211 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:40:28,212 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:40:28,305 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:40:29,306 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:30,306 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:32,307 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:35,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:37,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:38,773 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:40:38,773 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:40:48,324 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:40:48,324 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:40:48,325 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:40:49,087 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:40:49,314 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:40:49,314 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:51,315 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:53,316 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:53,986 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:40:53,987 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:40:55,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:40:57,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:08,433 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:41:08,434 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:41:08,434 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:41:09,213 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:41:09,214 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:41:09,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:41:09,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:11,323 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:13,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:15,325 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:17,326 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:19,167 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:41:24,407 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:41:24,408 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:41:28,528 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:41:28,529 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:41:28,529 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:41:29,330 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:41:29,330 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:31,331 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:33,332 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:35,333 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:37,334 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:39,537 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:41:39,537 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:41:48,656 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:41:48,657 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:41:48,657 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:41:49,247 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:41:49,338 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:41:49,339 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:51,339 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:53,340 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:54,725 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:41:54,726 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:41:55,341 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:41:57,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:08,786 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:42:08,786 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:42:08,787 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:42:09,346 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:42:09,947 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:42:09,948 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:42:10,347 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:11,347 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:13,348 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:15,349 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:17,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:19,328 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:42:25,131 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:42:25,131 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:42:28,882 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:42:28,882 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:42:28,882 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:42:29,354 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:42:29,354 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:30,355 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:31,355 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:33,356 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:35,357 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:37,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:40,266 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:42:40,266 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:42:48,927 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:42:48,928 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:42:48,928 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:42:49,362 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:42:49,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:49,405 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:42:50,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:52,364 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:54,365 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:55,402 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:42:55,403 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:42:56,366 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:42:58,367 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:09,101 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:43:09,102 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:43:09,102 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:43:09,372 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:43:10,372 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:10,616 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:43:10,616 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:43:12,373 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:14,374 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:16,375 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:18,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:19,485 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:43:25,767 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:43:25,768 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:43:29,232 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:43:29,232 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:43:29,233 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:43:29,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:43:30,381 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:32,382 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:34,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:36,384 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:38,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:40,899 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:43:40,900 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:43:49,385 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:43:49,385 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:43:49,386 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:43:49,389 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:43:49,565 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:43:50,390 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:52,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:54,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:56,169 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:43:56,169 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:43:56,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:43:58,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:09,557 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:44:09,557 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:44:09,558 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:44:10,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:44:10,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:11,374 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:44:11,374 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:44:12,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:14,400 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:16,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:18,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:19,645 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:44:26,523 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:44:26,523 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:44:29,630 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:44:29,631 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:44:29,631 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:44:30,406 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:44:30,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:32,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:34,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:36,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:38,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:41,657 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:44:41,658 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:44:49,725 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:44:49,743 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:44:49,743 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:44:49,743 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:44:50,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:44:50,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:51,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:53,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:55,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:56,794 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:44:56,795 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:44:57,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:44:59,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:09,877 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:45:09,878 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:45:09,878 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:45:10,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:45:11,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:11,974 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:45:11,975 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:45:13,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:15,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:17,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:19,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:19,807 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:45:27,143 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:45:27,143 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:45:29,953 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:45:29,954 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:45:29,954 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:45:30,431 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:45:31,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:33,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:35,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:37,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:39,435 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:42,272 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:45:42,272 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:45:49,886 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:45:50,045 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:45:50,046 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:45:50,046 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:45:50,440 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:45:51,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:53,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:55,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:57,402 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:45:57,402 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:45:57,444 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:45:59,445 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:10,155 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:46:10,156 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:46:10,156 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:46:10,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:46:11,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:12,576 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:46:12,577 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:46:13,450 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:15,451 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:17,452 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:19,966 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:46:27,745 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:46:27,746 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:46:30,210 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:46:30,211 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:46:30,211 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:46:30,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:46:31,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:33,458 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:35,459 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:37,460 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:39,461 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:42,878 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:46:42,878 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:46:50,046 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:46:50,325 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:46:50,326 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:46:50,326 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:46:50,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:46:51,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:53,466 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:56,468 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:46:58,015 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:46:58,016 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:46:58,469 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:00,470 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:10,473 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:47:10,474 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:47:10,475 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:47:10,475 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:47:11,475 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:12,475 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:13,192 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:47:13,193 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:47:14,476 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:16,477 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:18,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:20,124 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:47:28,331 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:47:28,332 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:47:30,587 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:47:30,587 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:47:30,588 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:47:31,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:47:31,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:32,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:34,485 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:36,486 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:38,487 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:43,463 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:47:43,463 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:47:50,192 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:47:50,802 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:47:50,802 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:47:50,803 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:47:51,492 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:47:51,492 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:52,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:54,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:56,495 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:58,496 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:47:58,596 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:47:58,596 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:48:10,872 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:48:10,873 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:48:10,873 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:48:11,501 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:48:12,502 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:13,830 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:48:13,830 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:48:14,503 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:16,504 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:18,505 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:20,262 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:48:28,980 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:48:28,980 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:48:30,985 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:48:30,986 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:48:30,986 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:48:31,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:48:32,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:34,512 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:36,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:38,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:44,112 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:48:44,113 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:48:50,332 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:48:51,097 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:48:51,097 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:48:51,098 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:48:51,519 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:48:52,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:54,521 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:56,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:58,523 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:48:59,244 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:48:59,244 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:49:11,189 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:49:11,190 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:49:11,190 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:49:11,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:49:12,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:14,384 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:49:14,384 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:49:14,529 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:17,531 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:19,532 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:20,404 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:49:29,517 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:49:29,518 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:49:31,297 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:49:31,298 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:49:31,298 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:49:31,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:49:32,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:33,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:35,539 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:37,540 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:39,278 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:49:39,279 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:49:39,279 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:49:39,541 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:49:41,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:43,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:44,723 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:49:44,723 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:49:45,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:47,545 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:49:50,473 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:49:59,394 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:49:59,395 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:49:59,395 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:49:59,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:49:59,893 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:49:59,893 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:50:00,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:01,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:03,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:05,553 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:07,554 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:15,029 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:50:15,029 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:50:19,442 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:50:19,443 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:50:19,443 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:50:19,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:50:20,547 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:50:20,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:21,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:23,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:25,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:27,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:30,165 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:50:30,165 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:50:39,515 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:50:39,516 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:50:39,516 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:50:39,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:50:40,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:41,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:43,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:45,396 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:50:45,396 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:50:45,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:47,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:50:50,620 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:50:59,638 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:50:59,638 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:50:59,639 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:51:00,575 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:51:00,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:00,604 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:51:00,605 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:51:01,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:03,577 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:05,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:07,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:15,735 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:51:15,736 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:51:19,795 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:51:19,795 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:51:19,796 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:51:20,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:51:20,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:20,694 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:51:21,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:23,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:25,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:27,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:30,867 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:51:30,867 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:51:39,932 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:51:39,933 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:51:39,933 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:51:40,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:51:41,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:43,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:46,066 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:51:46,067 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:51:46,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:48,596 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:51:50,766 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:52:00,053 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:52:00,053 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:52:00,054 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:52:00,600 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:52:01,275 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:52:01,275 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:52:01,601 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:02,601 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:04,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:06,603 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:08,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:16,434 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:52:16,434 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:52:20,196 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:52:20,196 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:52:20,197 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:52:20,609 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:52:20,840 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:52:21,609 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:22,610 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:24,610 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:26,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:28,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:31,572 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:52:31,573 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:52:40,265 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:52:40,266 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:52:40,266 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:52:40,617 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:52:41,617 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:42,618 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:44,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:46,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:46,707 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:52:46,707 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:52:48,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:52:50,917 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:53:00,450 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:53:00,451 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:53:00,451 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:53:00,625 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:53:01,625 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:01,902 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:53:01,902 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:53:02,626 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:04,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:06,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:08,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:17,038 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:53:17,038 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:53:20,531 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:53:20,531 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:53:20,532 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:53:20,633 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:53:20,991 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:53:21,633 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:22,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:24,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:26,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:28,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:32,175 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:53:32,175 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:53:40,620 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:53:40,620 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:53:40,621 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:53:40,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:53:41,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:42,642 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:44,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:46,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:47,320 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:53:47,320 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:53:48,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:53:51,065 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:54:00,741 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:54:00,741 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:54:00,742 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:54:01,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:54:01,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:02,462 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:54:02,463 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:54:02,650 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:04,650 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:07,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:09,653 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:17,597 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:54:17,598 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:54:20,855 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:54:20,855 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:54:20,856 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:54:21,143 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:54:21,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:54:21,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:23,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:25,659 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:27,660 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:29,661 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:32,728 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:54:32,728 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:54:40,936 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:54:40,936 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:54:40,937 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:54:41,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:54:42,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:43,667 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:45,668 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:47,669 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:47,861 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:54:47,862 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:54:49,669 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:54:51,221 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:55:01,091 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:55:01,092 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:55:01,092 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:55:01,674 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:55:02,675 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:03,055 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:55:03,056 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:55:03,675 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:05,676 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:07,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:09,678 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:18,236 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:55:18,237 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:55:21,227 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:55:21,227 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:55:21,229 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:55:21,302 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:55:21,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:55:22,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:23,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:25,684 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:27,685 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:29,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:33,370 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:55:33,371 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:55:41,352 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:55:41,352 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:55:41,353 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:55:41,691 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:55:42,691 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:43,692 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:45,693 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:47,693 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:48,547 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:55:48,548 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:55:49,694 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:55:51,381 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:56:01,479 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:56:01,480 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:56:01,480 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:56:01,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:56:02,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:03,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:03,770 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:56:03,770 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:56:05,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:07,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:09,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:18,919 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:56:18,920 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:56:21,462 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:56:21,614 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:56:21,615 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:56:21,615 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:56:21,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:56:22,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:23,708 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:26,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:28,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:30,711 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:34,053 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:56:34,053 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:56:41,715 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:56:41,715 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:56:41,716 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:56:41,717 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:56:42,717 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:44,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:46,719 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:48,720 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:49,185 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:56:49,186 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:56:50,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:56:51,540 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:57:01,819 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:57:01,820 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:57:01,820 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:57:02,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:57:02,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:04,382 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:57:04,382 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:57:04,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:06,727 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:08,728 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:10,729 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:19,569 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:57:19,569 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:57:21,621 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:57:21,878 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:57:21,878 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:57:21,879 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:57:22,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:57:22,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:24,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:26,735 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:28,736 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:30,737 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:34,703 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:57:34,703 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:57:41,959 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:57:41,959 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:57:41,960 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:57:42,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:57:42,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:44,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:46,744 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:48,745 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:49,833 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:57:49,833 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:57:50,746 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:57:51,701 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:58:02,125 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:58:02,125 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:58:02,126 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:58:02,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:58:02,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:04,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:05,048 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:58:05,048 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:58:06,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:09,753 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:11,754 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:20,187 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:58:20,188 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:58:21,779 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:58:22,206 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:58:22,206 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:58:22,207 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:58:22,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:58:23,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:25,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:27,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:29,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:31,762 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:35,319 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:58:35,319 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:58:42,294 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:58:42,294 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:58:42,295 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:58:42,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:58:43,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:45,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:47,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:49,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:50,451 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:58:50,451 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:58:51,770 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:58:51,860 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:59:02,416 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:59:02,416 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:59:02,417 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:59:02,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:59:03,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:05,608 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:59:05,608 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:59:05,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:07,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:09,778 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:11,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:20,739 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:59:20,740 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:59:21,941 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 07:59:22,706 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:59:22,707 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:59:22,707 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:59:22,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:59:23,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:25,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:27,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:29,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:31,787 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:35,870 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:59:35,870 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:59:42,751 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 07:59:42,752 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 07:59:42,752 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 07:59:42,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 07:59:43,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:45,793 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:47,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:49,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:51,001 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 07:59:51,001 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 07:59:51,796 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 07:59:52,012 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:00:02,871 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:00:02,872 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:00:02,872 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:00:03,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:00:03,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:05,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:06,171 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:00:06,172 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:00:07,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:09,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:11,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:21,345 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:00:21,346 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:00:22,083 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:00:22,940 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:00:22,941 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:00:22,941 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:00:23,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:00:23,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:25,810 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:28,811 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:30,812 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:32,813 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:36,479 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:00:36,480 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:00:43,038 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:00:43,038 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:00:43,039 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:00:43,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:00:43,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:44,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:46,818 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:48,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:50,820 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:00:51,611 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:00:51,611 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:00:52,160 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:00:52,821 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:03,149 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:01:03,149 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:01:03,150 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:01:03,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:01:03,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:04,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:06,746 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:01:06,747 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:01:06,827 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:08,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:10,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:21,900 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:01:21,900 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:01:22,238 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:01:23,249 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:01:23,249 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:01:23,250 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:01:23,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:01:23,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:24,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:26,835 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:28,836 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:30,837 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:37,033 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:01:37,033 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:01:43,434 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:01:43,435 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:01:43,435 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:01:43,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:01:44,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:46,844 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:48,844 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:50,845 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:01:52,173 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:01:52,173 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:01:52,314 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:01:52,846 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:03,541 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:02:03,542 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:02:03,543 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:02:03,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:02:04,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:06,852 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:07,359 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:02:07,359 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:02:09,853 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:11,854 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:13,855 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:22,394 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:02:22,495 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:02:22,496 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:02:23,695 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:02:23,695 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:02:23,696 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:02:23,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:02:24,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:25,860 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:27,861 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:29,861 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:31,862 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:37,626 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:02:37,626 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:02:43,749 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:02:43,750 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:02:43,750 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:02:43,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:02:44,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:45,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:47,869 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:49,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:51,871 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:02:52,475 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:02:52,759 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:02:52,759 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:02:53,872 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:03,848 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:03:03,848 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:03:03,849 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:03:03,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:03:04,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:05,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:07,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:07,913 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:03:07,913 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:03:09,878 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:11,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:22,556 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:03:23,235 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:03:23,236 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:03:23,975 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:03:23,975 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:03:23,976 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:03:24,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:03:24,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:25,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:27,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:29,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:31,888 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:38,366 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:03:38,367 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:03:44,060 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:03:44,060 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:03:44,060 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:03:44,894 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:03:44,894 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:45,894 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:47,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:49,896 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:51,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:03:52,637 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:03:53,498 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:03:53,498 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:04:04,188 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:04:04,188 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:04:04,189 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:04:04,902 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:04:04,902 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:05,902 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:07,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:08,656 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:04:08,656 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:04:09,904 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:11,905 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:22,716 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:04:23,797 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:04:23,797 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:04:24,301 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:04:24,302 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:04:24,302 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:04:24,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:04:24,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:25,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:27,912 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:30,913 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:32,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:38,928 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:04:38,928 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:04:44,410 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:04:44,410 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:04:44,410 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:04:44,919 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:04:44,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:46,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:48,921 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:50,922 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:52,795 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:04:52,923 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:04:54,060 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:04:54,061 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:05:04,470 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:05:04,470 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:05:04,471 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:05:04,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:05:05,929 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:06,929 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:08,930 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:09,198 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:05:09,198 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:05:10,931 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:12,932 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:22,876 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:05:24,380 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:05:24,381 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:05:24,570 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:05:24,570 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:05:24,571 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:05:24,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:05:25,938 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:26,938 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:28,939 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:30,940 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:32,941 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:39,516 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:05:39,516 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:05:44,740 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:05:44,740 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:05:44,741 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:05:44,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:05:45,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:46,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:48,947 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:50,949 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:52,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:05:52,956 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:05:54,645 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:05:54,646 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:06:04,843 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:06:04,843 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:06:04,844 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:06:04,955 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:06:05,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:06,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:08,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:09,810 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:06:09,810 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:06:10,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:12,959 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:23,038 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:06:24,897 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:06:24,897 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:06:24,898 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:06:24,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:06:25,097 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:06:25,098 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:06:25,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:26,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:28,966 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:30,967 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:32,968 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:40,260 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:06:40,260 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:06:44,949 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:06:44,949 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:06:44,950 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:06:44,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:06:45,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:46,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:48,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:50,975 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:52,976 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:06:53,116 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:06:55,392 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:06:55,393 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:07:05,060 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:07:05,061 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:07:05,062 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:07:05,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:07:05,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:07,983 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:09,983 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:10,570 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:07:10,571 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:07:11,984 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:13,985 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:23,194 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:07:25,185 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:07:25,185 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:07:25,186 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:07:25,743 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:07:25,744 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:07:25,990 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:07:25,991 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:27,991 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:29,992 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:31,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:33,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:40,944 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:07:40,944 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:07:45,331 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:07:45,332 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:07:45,332 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:07:45,999 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:07:47,000 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:48,000 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:50,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:52,002 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:53,270 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:07:54,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:07:56,076 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:07:56,076 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:08:05,398 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:08:05,399 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:08:05,399 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:08:06,007 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:08:07,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:08,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:10,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:11,214 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:08:11,214 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:08:12,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:14,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:23,347 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:08:25,492 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:08:25,493 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:08:25,493 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:08:26,016 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:08:26,389 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:08:26,389 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:08:27,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:28,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:30,018 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:32,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:34,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:41,524 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:08:41,524 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:08:45,551 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:08:45,551 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:08:45,552 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:08:46,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:08:47,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:48,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:50,026 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:52,027 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:53,430 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:08:54,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:08:56,654 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:08:56,655 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:09:05,629 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:09:05,629 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:09:05,630 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:09:06,035 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:09:06,035 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:07,035 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:09,036 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:11,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:11,855 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:09:11,855 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:09:13,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:15,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:23,510 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:09:25,724 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:09:25,724 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:09:25,725 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:09:26,043 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:09:27,044 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:27,078 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:09:27,079 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:09:29,045 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:31,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:33,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:35,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:42,278 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:09:42,278 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:09:45,815 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:09:45,815 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:09:45,816 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:09:46,052 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:09:47,053 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:49,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:51,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:53,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:53,592 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:09:55,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:09:57,417 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:09:57,418 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:10:05,904 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:10:05,904 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:10:05,905 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:10:06,061 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:10:07,061 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:09,062 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:11,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:12,549 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:10:12,550 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:10:13,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:15,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:23,673 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:10:26,018 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:10:26,018 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:10:26,019 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:10:26,070 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:10:27,070 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:27,700 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:10:27,700 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:10:29,071 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:31,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:33,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:35,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:42,878 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:10:42,878 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:10:46,102 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:10:46,102 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:10:46,103 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:10:47,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:10:47,080 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:49,080 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:51,081 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:53,082 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:53,752 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:10:55,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:10:58,007 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:10:58,008 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:11:06,279 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:11:06,279 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:11:06,280 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:11:07,088 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:11:07,088 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:10,089 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:12,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:13,141 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:11:13,141 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:11:14,091 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:16,092 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:23,830 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:11:26,373 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:11:26,373 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:11:26,374 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:11:27,096 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:11:27,097 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:28,097 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:28,342 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:11:28,343 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:11:30,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:32,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:34,099 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:36,100 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:43,510 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:11:43,510 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:11:46,422 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:11:46,423 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:11:46,423 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:11:47,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:11:47,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:48,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:50,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:52,107 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:53,908 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:11:54,108 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:11:58,640 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:11:58,641 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:12:06,513 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:12:06,513 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:12:06,514 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:12:07,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:12:08,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:10,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:12,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:13,779 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:12:13,780 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:12:14,116 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:23,977 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:12:26,611 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:12:26,611 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:12:26,611 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:12:27,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:12:28,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:28,968 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:12:28,969 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:12:30,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:32,124 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:34,124 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:44,107 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:12:44,107 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:12:46,791 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:12:46,792 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:12:46,792 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:12:47,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:12:48,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:50,131 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:53,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:54,046 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:12:55,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:12:59,238 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:12:59,238 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:13:06,870 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:13:06,870 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:13:06,871 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:13:07,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:13:08,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:09,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:11,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:13,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:14,409 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:13:14,410 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:13:15,142 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:24,115 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:13:26,961 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:13:26,961 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:13:26,962 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:13:27,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:13:28,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:29,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:29,611 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:13:29,611 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:13:31,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:33,149 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:35,150 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:44,760 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:13:44,761 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:13:47,068 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:13:47,068 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:13:47,069 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:13:47,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:13:48,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:49,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:51,158 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:53,159 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:54,184 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:13:55,160 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:57,161 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:13:59,894 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:13:59,894 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:14:07,163 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:14:07,164 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:14:07,164 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:14:07,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:14:08,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:09,166 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:11,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:13,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:15,032 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:14:15,032 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:14:15,169 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:24,254 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:14:27,270 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:14:27,270 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:14:27,271 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:14:28,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:14:28,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:29,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:30,222 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:14:30,222 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:14:31,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:33,176 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:35,177 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:45,364 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:14:45,365 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:14:47,342 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:14:47,342 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:14:47,343 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:14:48,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:14:48,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:49,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:51,184 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:53,185 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:14:54,324 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:14:55,186 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:00,496 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:15:00,496 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:15:07,444 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:15:07,445 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:15:07,445 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:15:08,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:15:09,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:11,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:14,193 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:15,632 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:15:15,633 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:15:16,194 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:24,395 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:15:27,549 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:15:27,549 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:15:27,550 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:15:28,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:15:29,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:30,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:30,835 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:15:30,835 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:15:32,201 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:34,202 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:36,203 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:46,032 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:15:46,032 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:15:47,662 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:15:47,662 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:15:47,663 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:15:48,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:15:49,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:50,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:52,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:54,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:15:54,463 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:15:56,211 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:01,167 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:16:01,167 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:16:07,766 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:16:07,767 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:16:07,767 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:16:08,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:16:09,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:10,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:12,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:14,219 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:16,220 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:16,309 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:16:16,309 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:16:24,533 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:16:27,905 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:16:27,905 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:16:27,906 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:16:28,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:16:29,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:30,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:31,481 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:16:31,482 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:16:32,227 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:34,228 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:36,229 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:46,614 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:16:46,615 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:16:47,952 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:16:47,954 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:16:47,954 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:16:48,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:16:49,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:50,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:52,236 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:54,237 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:16:54,603 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:16:56,238 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:01,754 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:17:01,754 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:17:08,047 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:17:08,048 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:17:08,048 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:17:08,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:17:09,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:10,244 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:12,244 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:14,246 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:16,246 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:16,886 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:17:16,887 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:17:24,676 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:17:28,149 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:17:28,150 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:17:28,150 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:17:28,251 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:17:29,251 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:30,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:32,021 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:17:32,022 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:17:32,253 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:34,254 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:36,255 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:47,157 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:17:47,158 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:17:48,310 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:17:48,310 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:17:48,312 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:17:49,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:17:49,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:50,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:52,261 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:54,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:17:54,748 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:17:56,263 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:02,291 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:18:02,291 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:18:08,403 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:18:08,404 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:18:08,404 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:18:09,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:18:09,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:10,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:12,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:15,271 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:17,272 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:17,427 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:18:17,427 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:18:24,821 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:18:28,509 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:18:28,510 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:18:28,510 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:18:29,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:18:29,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:31,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:32,595 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:18:32,595 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:18:33,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:35,279 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:37,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:47,727 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:18:47,727 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:18:48,658 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:18:48,659 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:18:48,659 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:18:49,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:18:50,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:51,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:53,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:54,890 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:18:55,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:18:57,288 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:02,859 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:19:02,859 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:19:08,775 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:19:08,775 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:19:08,776 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:19:09,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:19:10,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:11,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:13,295 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:15,296 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:17,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:17,991 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:19:17,991 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:19:24,962 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:19:28,904 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:19:28,905 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:19:28,905 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:19:29,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:19:30,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:31,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:33,186 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:19:33,186 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:19:33,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:35,303 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:37,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:48,480 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:19:48,481 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:19:48,983 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:19:48,983 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:19:48,984 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:19:49,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:19:50,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:51,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:53,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:55,033 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:19:55,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:19:57,313 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:03,612 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:20:03,613 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:20:09,343 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:20:09,343 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:20:09,344 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:20:10,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:20:11,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:12,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:13,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:15,320 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:17,321 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:18,744 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:20:18,744 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:20:25,106 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:20:29,466 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:20:29,467 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:20:29,467 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:20:30,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:20:30,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:31,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:33,328 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:33,894 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:20:33,894 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:20:36,329 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:38,331 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:49,059 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:20:49,059 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:20:49,588 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:20:49,588 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:20:49,589 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:20:50,336 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:20:50,336 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:52,337 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:54,338 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:55,178 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:20:56,338 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:20:58,339 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:04,199 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:21:04,199 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:21:09,682 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:21:09,683 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:21:09,683 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:21:10,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:21:12,345 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:14,346 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:16,347 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:18,347 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:19,332 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:21:19,332 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:21:25,258 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:21:29,799 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:21:29,800 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:21:29,800 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:21:30,352 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:21:30,352 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:32,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:34,354 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:34,532 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:21:34,533 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:21:36,355 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:38,356 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:49,673 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:21:49,674 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:21:49,889 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:21:49,889 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:21:49,890 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:21:50,361 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:21:50,361 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:52,362 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:54,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:55,334 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:21:56,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:21:58,364 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:04,805 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:22:04,805 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:22:09,961 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:22:09,961 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:22:09,962 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:22:10,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:22:12,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:14,370 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:16,371 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:18,372 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:19,936 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:22:19,936 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:22:25,412 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:22:30,071 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:22:30,071 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:22:30,072 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:22:30,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:22:33,378 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:35,081 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:22:35,082 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:22:35,378 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:37,379 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:39,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:50,165 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:22:50,165 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:22:50,170 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:22:50,260 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:22:50,260 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:22:50,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:22:51,384 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:53,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:55,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:55,491 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:22:57,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:22:59,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:05,406 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:23:05,406 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:23:10,257 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:23:10,258 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:23:10,258 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:23:10,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:23:11,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:13,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:15,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:17,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:19,394 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:20,537 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:23:20,537 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:23:25,569 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:23:30,369 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:23:30,369 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:23:30,370 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:23:30,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:23:31,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:33,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:35,400 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:35,670 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:23:35,671 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:23:37,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:39,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:50,404 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:23:50,405 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:23:50,405 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:23:50,405 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:23:50,919 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:23:50,919 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:23:51,406 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:53,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:55,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:55,647 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:23:57,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:23:59,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:06,062 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:24:06,062 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:24:10,495 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:24:10,495 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:24:10,496 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:24:11,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:24:11,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:12,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:14,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:16,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:18,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:20,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:21,196 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:24:21,196 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:24:25,724 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:24:30,598 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:24:30,599 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:24:30,599 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:24:31,421 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:24:31,421 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:32,422 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:34,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:36,339 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:24:36,339 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:24:36,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:38,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:50,643 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:24:50,644 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:24:50,644 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:24:51,429 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:24:51,430 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:51,600 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:24:51,600 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:24:52,430 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:54,431 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:55,802 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:24:56,431 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:24:58,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:06,749 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:25:06,750 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:25:10,714 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:25:10,715 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:25:10,715 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:25:11,437 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:25:12,438 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:14,438 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:16,439 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:18,440 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:21,882 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:25:21,882 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:25:25,879 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:25:30,862 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:25:30,862 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:25:30,863 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:25:31,444 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:25:32,445 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:34,446 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:36,446 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:37,022 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:25:37,022 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:25:38,447 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:50,945 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:25:50,945 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:25:50,946 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:25:51,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:25:52,190 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:25:52,190 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:25:52,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:54,454 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:55,957 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:25:56,455 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:25:58,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:07,331 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:26:07,331 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:26:11,040 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:26:11,040 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:26:11,041 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:26:11,462 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:26:12,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:15,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:17,466 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:19,467 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:22,462 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:26:22,462 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:26:26,035 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:26:31,186 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:26:31,187 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:26:31,187 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:26:31,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:26:32,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:33,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:35,474 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:37,475 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:37,621 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:26:37,621 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:26:39,476 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:51,270 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:26:51,271 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:26:51,271 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:26:51,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:26:52,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:52,811 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:26:52,811 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:26:53,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:55,482 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:56,114 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:26:57,483 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:26:59,483 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:08,000 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:27:08,000 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:27:11,441 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:27:11,441 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:27:11,441 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:27:11,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:27:12,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:13,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:15,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:17,490 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:19,491 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:23,132 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:27:23,133 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:27:26,193 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:27:31,534 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:27:31,534 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:27:31,535 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:27:32,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:27:32,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:33,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:35,498 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:37,499 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:38,270 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:27:38,270 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:27:39,500 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:51,617 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:27:51,617 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:27:51,618 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:27:52,506 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:27:52,506 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:53,416 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:27:53,416 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:27:53,506 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:55,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:56,273 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:27:57,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:27:59,508 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:08,570 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:28:08,571 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:28:11,779 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:28:11,779 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:28:11,779 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:28:12,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:28:12,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:13,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:15,515 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:17,516 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:19,517 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:23,703 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:28:23,703 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:28:26,352 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:28:31,898 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:28:31,898 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:28:31,898 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:28:32,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:28:32,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:33,523 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:35,524 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:37,525 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:38,910 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:28:38,911 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:28:40,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:52,008 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:28:52,008 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:28:52,009 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:28:52,532 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:28:53,532 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:54,110 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:28:54,110 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:28:54,532 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:56,431 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:28:56,533 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:28:58,534 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:00,535 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:09,254 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:29:09,254 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:29:12,153 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:29:12,154 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:29:12,154 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:29:12,540 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:29:13,541 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:14,541 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:16,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:18,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:20,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:24,387 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:29:24,388 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:29:26,508 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:29:32,251 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:29:32,251 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:29:32,252 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:29:32,549 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:29:33,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:34,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:36,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:38,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:39,526 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:29:39,526 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:29:40,553 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:52,328 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:29:52,328 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:29:52,329 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:29:52,558 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:29:53,558 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:54,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:54,663 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:29:54,663 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:29:56,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:29:56,591 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:29:58,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:00,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:09,822 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:30:09,823 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:30:12,379 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:30:12,380 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:30:12,380 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:30:12,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:30:13,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:14,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:16,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:18,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:20,571 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:24,954 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:30:24,954 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:30:26,672 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:30:32,493 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:30:32,493 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:30:32,494 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:30:32,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:30:33,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:34,577 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:36,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:38,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:40,085 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:30:40,085 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:30:40,580 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:52,613 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:30:52,613 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:30:52,614 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:30:53,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:30:53,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:54,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:55,238 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:30:55,238 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:30:56,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:30:56,750 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:30:58,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:00,588 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:10,372 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:31:10,373 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:31:12,662 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:31:12,662 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:31:12,662 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:31:13,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:31:13,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:14,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:17,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:19,596 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:21,597 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:25,502 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:31:25,502 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:31:26,826 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:31:32,736 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:31:32,737 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:31:32,737 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:31:33,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:31:33,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:35,603 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:37,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:39,605 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:40,640 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:31:40,640 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:31:41,606 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:52,893 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:31:52,893 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:31:52,893 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:31:53,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:31:53,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:55,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:55,800 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:31:55,800 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:31:56,896 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:31:57,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:31:59,615 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:01,616 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:10,945 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:32:10,945 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:32:13,099 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:32:13,099 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:32:13,100 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:32:13,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:32:13,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:15,622 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:17,623 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:19,624 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:21,625 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:26,076 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:32:26,077 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:32:26,969 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:32:33,250 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:32:33,251 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:32:33,251 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:32:33,630 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:32:34,630 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:35,631 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:37,632 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:39,633 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:41,207 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:32:41,207 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:32:41,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:53,328 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:32:53,329 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:32:53,329 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:32:53,639 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:32:55,640 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:56,404 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:32:56,404 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:32:56,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:57,043 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:32:57,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:32:59,642 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:01,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:11,595 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:33:11,596 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:33:13,423 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:33:13,423 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:33:13,424 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:33:13,648 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:33:14,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:15,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:17,650 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:19,651 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:21,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:26,726 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:33:26,726 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:33:27,117 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:33:33,526 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:33:33,526 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:33:33,526 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:33:33,656 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:33:34,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:35,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:37,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:39,659 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:41,660 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:41,855 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:33:41,855 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:33:53,638 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:33:53,638 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:33:53,639 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:33:53,664 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:33:54,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:55,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:57,001 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:33:57,001 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:33:57,193 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:33:57,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:33:59,667 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:01,668 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:12,166 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:34:12,166 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:34:13,778 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:34:13,779 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:34:13,779 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:34:14,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:34:15,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:16,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:17,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:20,675 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:22,676 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:27,274 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:34:27,306 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:34:27,307 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:34:33,870 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:34:33,870 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:34:33,871 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:34:34,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:34:34,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:36,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:38,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:40,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:42,442 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:34:42,443 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:34:42,684 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:53,939 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:34:53,939 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:34:53,939 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:34:54,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:34:54,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:56,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:34:57,352 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:34:57,580 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:34:57,581 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:34:58,691 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:00,692 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:02,693 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:12,715 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:35:12,716 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:35:14,042 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:35:14,042 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:35:14,042 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:35:14,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:35:14,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:16,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:18,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:20,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:22,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:27,429 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:35:27,857 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:35:27,858 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:35:34,125 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:35:34,125 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:35:34,126 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:35:34,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:35:34,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:36,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:38,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:40,708 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:42,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:42,994 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:35:42,994 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:35:54,198 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:35:54,198 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:35:54,199 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:35:54,713 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:35:55,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:56,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:35:57,504 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:35:58,133 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:35:58,133 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:35:58,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:00,716 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:02,717 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:13,288 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:36:13,288 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:36:14,267 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:36:14,268 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:36:14,268 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:36:14,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:36:15,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:16,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:18,723 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:20,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:22,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:27,580 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:36:28,423 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:36:28,424 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:36:34,352 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:36:34,352 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:36:34,353 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:36:34,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:36:34,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:35,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:36,731 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:38,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:40,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:42,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:43,563 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:36:43,564 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:36:54,484 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:36:54,485 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:36:54,485 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:36:54,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:36:55,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:57,658 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:36:57,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:36:58,721 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:36:58,721 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:36:59,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:01,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:03,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:13,947 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:37:13,948 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:37:14,575 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:37:14,575 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:37:14,575 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:37:14,748 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:37:15,749 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:17,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:19,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:21,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:23,753 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:27,735 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:37:29,077 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:37:29,077 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:37:34,718 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:37:34,719 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:37:34,719 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:37:34,757 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:37:35,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:37,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:39,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:41,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:43,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:44,210 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:37:44,210 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:37:54,808 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:37:54,809 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:37:54,809 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:37:55,765 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:37:55,765 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:57,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:37:57,813 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:37:59,370 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:37:59,370 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:37:59,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:01,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:03,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:14,590 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:38:14,590 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:38:14,848 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:38:14,848 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:38:14,848 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:38:15,772 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:38:15,773 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:17,773 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:19,774 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:21,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:23,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:27,891 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:38:29,722 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:38:29,723 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:38:34,923 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:38:34,924 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:38:34,924 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:38:35,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:38:35,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:37,780 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:39,781 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:41,781 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:43,782 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:44,855 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:38:44,855 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:38:55,059 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:38:55,059 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:38:55,059 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:38:55,787 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:38:55,787 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:57,788 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:38:57,969 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:38:59,788 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:00,014 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:39:00,014 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:39:01,789 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:03,790 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:15,174 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:39:15,174 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:39:15,175 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:39:15,243 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:39:15,243 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:39:15,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:39:15,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:16,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:18,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:20,796 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:22,797 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:24,797 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:28,047 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:39:30,490 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:39:30,490 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:39:35,251 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:39:35,251 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:39:35,252 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:39:35,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:39:36,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:38,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:40,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:42,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:44,805 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:45,621 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:39:45,621 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:39:55,368 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:39:55,368 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:39:55,369 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:39:55,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:39:56,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:39:58,124 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:39:58,810 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:00,763 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:40:00,763 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:40:00,811 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:02,811 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:04,812 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:15,471 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:40:15,471 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:40:15,471 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:40:15,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:40:15,953 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:40:15,953 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:40:16,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:18,818 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:20,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:22,820 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:24,820 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:28,202 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:40:31,108 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:40:31,108 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:40:35,550 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:40:35,551 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:40:35,551 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:40:35,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:40:36,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:38,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:40,827 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:42,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:44,829 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:46,238 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:40:46,238 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:40:55,646 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:40:55,646 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:40:55,647 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:40:55,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:40:56,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:40:58,280 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:40:58,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:00,835 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:01,367 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:41:01,368 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:41:02,836 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:04,837 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:15,809 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:41:15,809 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:41:15,810 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:41:15,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:41:16,539 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:41:16,539 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:41:16,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:18,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:20,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:22,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:24,844 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:28,359 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:41:31,695 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:41:31,695 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:41:35,994 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:41:35,994 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:41:35,995 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:41:36,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:41:36,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:38,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:40,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:42,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:44,852 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:46,829 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:41:46,830 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:41:56,117 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:41:56,117 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:41:56,118 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:41:56,856 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:41:56,856 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:41:58,435 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:41:58,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:00,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:01,983 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:42:01,983 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:42:02,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:04,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:16,229 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:42:16,230 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:42:16,230 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:42:16,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:42:16,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:17,124 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:42:17,124 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:42:17,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:19,864 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:21,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:23,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:25,866 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:28,513 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:42:32,263 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:42:32,263 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:42:36,383 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:42:36,384 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:42:36,384 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:42:36,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:42:37,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:39,871 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:41,871 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:43,872 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:45,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:47,394 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:42:47,394 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:42:56,428 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:42:56,429 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:42:56,429 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:42:56,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:42:57,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:42:58,587 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:42:59,878 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:01,878 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:02,529 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:43:02,529 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:43:03,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:05,880 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:16,571 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:43:16,571 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:43:16,572 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:43:16,883 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:43:17,718 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:43:17,718 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:43:17,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:19,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:21,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:23,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:25,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:28,658 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:43:32,924 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:43:32,924 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:43:36,635 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:43:36,635 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:43:36,636 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:43:36,891 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:43:37,891 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:39,892 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:41,892 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:43,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:45,894 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:48,056 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:43:48,056 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:43:56,731 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:43:56,732 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:43:56,732 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:43:56,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:43:57,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:43:58,728 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:43:59,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:01,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:03,189 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:44:03,190 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:44:03,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:05,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:16,843 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:44:16,844 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:44:16,844 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:44:16,905 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:44:17,905 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:18,451 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:44:18,451 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:44:18,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:20,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:22,907 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:24,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:26,909 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:28,799 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:44:33,637 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:44:33,638 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:44:36,975 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:44:36,976 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:44:36,976 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:44:37,912 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:44:37,913 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:38,913 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:40,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:42,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:44,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:46,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:48,772 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:44:48,772 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:44:57,127 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:44:57,127 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:44:57,128 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:44:57,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:44:57,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:44:58,877 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:44:58,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:00,921 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:02,922 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:03,901 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:45:03,902 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:45:04,922 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:17,219 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:45:17,220 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:45:17,220 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:45:17,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:45:17,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:18,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:19,081 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:45:19,082 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:45:20,929 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:22,929 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:24,930 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:28,954 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:45:34,223 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:45:34,223 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:45:37,283 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:45:37,284 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:45:37,284 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:45:37,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:45:37,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:38,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:40,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:42,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:44,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:49,356 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:45:49,357 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:45:57,375 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:45:57,375 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:45:57,376 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:45:57,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:45:57,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:58,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:45:59,032 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:46:00,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:02,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:04,489 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:46:04,490 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:46:04,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:17,456 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:46:17,456 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:46:17,457 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:46:17,949 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:46:18,949 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:19,653 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:46:19,653 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:46:21,951 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:23,952 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:25,952 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:29,110 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:46:35,598 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:46:35,598 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:46:37,552 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:46:37,552 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:46:37,552 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:46:37,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:46:38,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:39,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:41,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:43,959 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:45,960 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:50,783 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:46:50,783 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:46:57,642 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:46:57,642 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:46:57,642 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:46:57,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:46:58,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:46:59,187 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:46:59,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:01,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:03,966 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:05,915 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:47:05,915 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:47:05,967 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:17,764 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:47:17,765 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:47:17,765 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:47:17,971 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:47:18,972 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:19,972 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:21,066 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:47:21,067 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:47:21,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:23,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:25,975 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:29,265 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:47:36,200 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:47:36,200 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:47:37,813 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:47:37,813 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:47:37,814 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:47:37,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:47:38,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:39,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:41,981 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:43,981 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:45,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:51,331 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:47:51,332 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:47:57,929 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:47:57,929 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:47:57,930 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:47:57,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:47:58,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:47:59,343 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:47:59,987 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:01,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:03,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:05,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:06,463 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:48:06,463 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:48:18,074 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:48:18,074 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:48:18,074 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:48:18,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:48:18,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:19,995 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:21,600 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:48:21,601 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:48:21,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:23,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:25,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:29,421 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:48:36,758 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:48:36,758 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:48:38,172 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:48:38,173 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:48:38,173 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:48:39,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:48:40,002 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:41,002 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:42,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:44,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:46,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:48:51,889 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:48:51,890 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:48:58,284 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:48:58,284 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:48:58,285 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:48:59,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:48:59,497 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:49:00,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:01,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:03,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:05,012 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:07,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:07,026 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:49:07,026 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:49:18,408 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:49:18,409 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:49:18,409 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:49:19,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:49:21,018 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:22,180 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:49:22,181 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:49:23,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:25,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:27,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:29,576 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:49:37,413 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:49:37,413 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:49:38,527 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:49:38,527 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:49:38,527 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:49:39,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:49:39,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:41,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:43,026 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:45,027 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:47,028 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:52,544 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:49:52,544 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:49:58,638 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:49:58,639 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:49:58,639 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:49:59,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:49:59,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:49:59,654 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:50:01,034 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:03,035 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:05,035 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:07,036 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:07,676 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:50:07,677 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:50:18,750 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:50:18,751 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:50:18,751 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:50:19,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:50:20,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:21,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:22,819 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:50:22,819 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:50:23,043 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:25,044 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:27,045 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:29,732 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:50:37,968 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:50:37,968 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:50:38,824 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:50:38,824 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:50:38,825 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:50:39,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:50:40,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:41,050 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:43,051 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:45,051 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:47,052 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:50:53,101 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:50:53,102 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:50:58,881 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:50:58,882 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:50:58,882 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:50:59,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:50:59,809 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:51:00,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:01,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:04,059 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:06,060 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:08,061 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:08,235 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:51:08,236 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:51:18,994 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:51:18,994 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:51:18,995 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:51:19,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:51:20,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:22,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:23,365 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:51:23,365 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:51:24,067 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:26,068 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:28,069 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:29,887 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:51:38,523 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:51:38,523 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:51:39,117 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:51:39,117 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:51:39,117 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:51:40,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:51:42,075 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:44,075 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:46,076 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:48,077 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:51:53,665 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:51:53,666 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:51:59,202 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:51:59,203 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:51:59,203 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:51:59,966 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:52:00,082 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:52:02,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:04,084 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:06,085 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:08,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:08,796 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:52:08,796 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:52:19,237 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:52:19,237 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:52:19,238 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:52:20,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:52:20,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:22,091 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:23,981 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:52:23,981 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:52:24,092 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:26,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:28,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:30,043 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:52:39,129 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:52:39,130 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:52:39,356 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:52:39,356 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:52:39,357 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:52:40,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:52:40,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:42,099 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:44,099 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:46,100 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:48,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:52:54,259 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:52:54,259 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:52:59,443 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:52:59,443 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:52:59,444 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:53:00,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:53:00,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:00,120 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:53:02,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:04,107 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:06,108 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:08,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:09,390 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:53:09,390 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:53:19,586 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:53:19,586 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:53:19,587 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:53:20,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:53:20,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:22,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:24,542 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:53:24,542 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:53:25,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:27,116 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:29,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:30,198 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:53:39,677 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:53:39,678 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:53:39,806 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:53:39,806 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:53:39,806 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:53:40,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:53:41,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:42,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:43,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:45,124 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:47,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:49,126 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:53:54,810 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:53:54,810 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:53:59,808 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:53:59,808 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:53:59,809 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:54:00,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:54:00,275 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:54:02,131 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:03,131 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:05,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:07,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:09,134 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:09,945 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:54:09,946 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:54:19,913 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:54:19,913 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:54:19,914 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:54:20,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:54:20,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:21,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:23,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:25,128 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:54:25,128 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:54:25,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:27,142 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:29,142 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:30,353 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:54:40,015 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:54:40,016 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:54:40,016 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:54:40,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:54:40,259 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:54:40,259 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:54:41,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:43,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:45,149 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:47,150 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:49,150 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:54:55,390 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:54:55,391 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:55:00,124 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:55:00,125 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:55:00,125 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:55:00,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:55:00,426 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:55:01,155 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:03,155 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:05,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:07,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:09,158 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:10,521 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:55:10,522 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:55:20,231 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:55:20,232 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:55:20,232 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:55:21,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:55:21,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:23,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:25,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:25,654 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:55:25,655 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:55:27,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:29,166 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:30,500 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:55:40,297 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:55:40,297 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:55:40,298 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:55:40,974 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:55:40,975 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:55:41,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:55:41,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:43,171 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:45,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:47,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:49,173 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:55:56,157 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:55:56,157 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:56:00,347 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:56:00,347 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:56:00,348 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:56:00,576 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:56:01,177 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:56:01,178 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:03,178 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:06,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:08,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:10,181 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:11,287 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:56:11,287 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:56:20,439 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:56:20,439 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:56:20,440 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:56:21,185 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:56:21,186 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:22,186 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:24,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:26,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:26,420 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:56:26,421 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:56:28,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:30,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:30,653 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:56:40,477 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:56:40,478 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:56:40,478 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:56:41,193 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:56:41,193 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:42,062 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:56:42,063 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:56:42,194 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:44,195 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:46,195 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:48,196 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:56:57,232 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:56:57,233 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:57:00,544 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:57:00,545 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:57:00,545 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:57:00,731 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:57:01,201 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:57:02,202 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:03,202 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:04,203 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:06,204 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:08,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:10,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:12,366 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:57:12,367 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:57:20,671 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:57:20,672 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:57:20,672 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:57:21,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:57:22,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:23,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:24,211 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:26,212 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:27,500 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:57:27,500 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:57:28,213 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:30,213 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:30,809 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:57:40,705 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:57:40,706 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:57:40,706 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:57:41,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:57:41,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:42,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:42,685 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:57:42,685 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:57:44,219 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:46,220 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:48,221 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:57:58,360 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:57:58,360 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:58:00,795 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:58:00,795 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:58:00,796 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:58:00,887 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:58:01,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:58:01,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:02,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:05,227 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:07,228 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:09,229 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:13,491 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:58:13,491 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:58:20,979 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:58:20,979 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:58:20,980 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:58:21,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:58:22,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:23,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:25,236 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:27,236 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:28,623 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:58:28,623 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:58:29,237 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:30,965 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:58:41,064 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:58:41,064 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:58:41,065 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:58:41,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:58:43,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:43,759 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:58:43,760 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:58:44,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:45,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:47,244 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:49,245 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:58:59,170 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:58:59,171 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:59:01,065 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:59:01,126 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:59:01,127 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:59:01,127 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:59:01,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:59:02,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:03,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:05,251 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:07,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:09,253 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:14,302 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:59:14,302 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:59:21,235 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:59:21,235 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:59:21,236 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:59:21,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:59:22,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:23,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:25,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:27,259 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:29,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:29,437 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:59:29,437 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:59:31,141 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 08:59:41,289 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 08:59:41,290 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 08:59:41,290 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 08:59:42,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 08:59:43,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:44,266 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:44,658 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:59:44,658 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 08:59:45,266 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:47,267 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:49,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 08:59:59,811 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 08:59:59,811 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:00:01,219 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:00:01,433 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:00:01,433 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:00:01,433 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:00:02,272 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:00:03,272 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:04,273 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:05,273 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:07,274 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:09,275 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:14,942 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:00:14,942 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:00:21,503 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:00:21,503 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:00:21,504 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:00:22,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:00:22,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:23,281 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:25,282 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:27,282 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:29,283 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:30,075 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:00:30,075 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:00:31,296 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:00:41,595 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:00:41,595 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:00:41,595 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:00:42,288 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:00:42,288 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:43,288 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:45,240 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:00:45,240 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:00:45,289 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:47,290 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:00:49,291 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:00,405 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:01:00,405 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:01:01,374 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:01:01,687 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:01:01,688 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:01:01,688 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:01:02,296 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:01:03,296 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:04,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:05,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:07,298 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:09,299 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:15,538 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:01:15,538 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:01:21,780 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:01:21,781 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:01:21,781 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:01:22,303 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:01:22,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:23,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:25,305 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:28,306 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:30,307 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:30,669 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:01:30,669 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:01:31,452 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:01:41,853 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:01:41,853 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:01:41,854 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:01:42,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:01:42,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:44,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:45,911 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:01:45,912 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:01:46,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:48,313 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:01:50,314 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:01,047 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:02:01,047 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:02:01,530 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:02:01,954 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:02:01,955 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:02:01,955 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:02:02,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:02:03,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:04,320 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:06,321 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:08,321 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:10,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:16,182 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:02:16,182 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:02:22,045 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:02:22,045 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:02:22,046 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:02:22,326 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:02:23,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:24,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:26,328 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:28,329 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:30,330 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:31,313 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:02:31,313 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:02:31,607 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:02:42,107 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:02:42,108 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:02:42,108 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:02:42,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:02:43,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:44,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:46,336 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:46,509 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:02:46,509 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:02:48,337 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:02:50,338 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:01,667 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:03:01,668 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:03:01,800 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:03:02,154 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:03:02,155 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:03:02,155 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:03:02,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:03:03,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:04,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:06,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:08,345 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:10,346 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:16,802 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:03:16,802 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:03:22,256 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:03:22,256 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:03:22,257 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:03:22,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:03:23,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:24,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:26,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:28,352 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:30,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:31,763 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:03:31,933 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:03:31,933 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:03:42,359 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:03:42,359 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:03:42,363 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:03:43,357 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:03:43,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:44,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:46,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:47,082 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:03:47,083 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:03:48,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:03:50,361 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:01,840 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:04:02,258 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:04:02,259 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:04:02,437 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:04:02,437 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:04:02,438 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:04:03,365 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:04:03,366 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:04,366 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:06,366 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:08,367 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:10,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:17,394 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:04:17,395 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:04:22,547 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:04:22,548 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:04:22,550 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:04:23,373 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:04:23,373 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:24,373 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:26,374 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:28,375 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:30,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:31,918 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:04:32,650 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:04:32,651 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:04:42,634 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:04:42,634 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:04:42,635 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:04:43,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:04:43,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:44,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:46,381 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:47,780 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:04:47,780 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:04:48,382 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:04:50,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:01,996 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:05:02,680 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:05:02,681 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:05:02,681 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:05:02,960 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:05:02,960 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:05:03,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:05:03,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:04,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:06,388 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:08,389 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:10,390 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:18,127 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:05:18,128 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:05:22,750 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:05:22,751 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:05:22,751 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:05:23,394 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:05:23,394 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:25,395 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:27,396 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:29,397 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:31,397 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:32,074 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:05:33,258 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:05:33,259 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:05:42,838 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:05:42,838 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:05:42,838 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:05:43,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:05:43,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:45,403 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:47,403 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:48,396 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:05:48,397 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:05:49,404 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:05:51,405 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:02,151 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:06:02,990 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:06:02,991 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:06:02,991 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:06:03,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:06:03,579 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:06:03,580 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:06:04,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:05,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:07,411 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:09,412 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:11,413 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:18,714 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:06:18,714 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:06:23,038 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:06:23,038 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:06:23,039 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:06:23,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:06:24,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:25,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:27,419 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:29,420 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:31,421 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:32,228 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:06:33,845 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:06:33,845 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:06:43,101 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:06:43,101 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:06:43,102 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:06:43,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:06:44,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:45,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:47,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:48,982 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:06:48,982 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:06:49,428 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:06:51,428 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:02,306 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:07:03,211 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:07:03,212 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:07:03,212 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:07:03,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:07:04,121 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:07:04,122 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:07:04,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:05,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:07,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:09,435 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:11,436 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:19,258 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:07:19,259 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:07:23,274 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:07:23,274 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:07:23,274 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:07:23,440 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:07:24,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:25,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:27,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:29,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:31,444 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:32,383 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:07:34,390 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:07:34,390 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:07:43,378 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:07:43,378 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:07:43,379 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:07:43,448 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:07:44,448 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:45,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:47,450 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:49,451 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:07:49,581 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:07:49,581 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:07:51,451 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:02,462 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:08:03,447 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:08:03,447 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:08:03,448 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:08:03,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:08:04,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:04,719 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:08:04,719 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:08:05,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:07,458 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:09,458 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:11,459 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:19,854 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:08:19,854 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:08:23,540 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:08:23,540 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:08:23,541 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:08:24,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:08:24,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:25,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:28,466 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:30,467 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:32,467 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:32,540 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:08:34,987 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:08:34,987 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:08:43,617 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:08:43,618 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:08:43,618 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:08:44,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:08:44,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:46,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:48,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:50,172 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:08:50,172 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:08:50,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:08:52,474 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:02,618 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:09:03,663 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:09:03,663 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:09:03,663 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:09:04,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:09:04,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:05,395 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:09:05,396 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:09:06,479 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:08,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:10,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:12,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:20,551 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:09:20,551 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:09:23,756 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:09:23,756 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:09:23,757 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:09:24,485 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:09:25,486 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:26,486 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:28,487 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:30,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:32,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:32,697 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:09:35,682 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:09:35,683 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:09:43,895 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:09:43,896 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:09:43,897 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:09:44,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:09:45,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:46,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:48,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:50,495 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:09:50,857 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:09:50,857 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:09:52,496 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:02,775 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:10:03,972 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:10:03,972 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:10:03,972 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:10:04,500 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:10:05,501 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:06,090 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:10:06,091 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:10:06,501 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:08,502 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:10,502 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:12,503 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:21,265 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:10:21,265 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:10:24,045 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:10:24,046 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:10:24,046 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:10:24,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:10:24,508 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:25,508 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:26,508 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:28,509 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:30,510 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:32,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:32,853 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:10:36,395 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:10:36,396 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:10:44,097 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:10:44,097 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:10:44,098 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:10:44,517 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:10:44,517 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:45,517 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:46,518 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:48,519 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:50,519 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:10:51,524 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:10:51,525 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:10:52,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:02,934 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:11:04,255 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:11:04,256 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:11:04,256 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:11:04,525 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:11:05,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:06,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:06,677 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:11:06,678 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:11:09,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:11,529 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:13,530 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:21,816 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:11:21,816 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:11:24,348 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:11:24,348 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:11:24,349 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:11:24,534 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:11:25,535 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:27,536 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:29,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:31,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:33,014 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:11:33,539 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:36,978 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:11:36,979 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:11:44,542 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:11:44,543 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:11:44,543 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:11:44,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:11:45,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:47,545 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:49,546 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:51,547 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:11:52,113 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:11:52,113 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:11:53,547 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:03,094 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:12:04,664 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:12:04,665 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:12:04,665 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:12:05,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:12:05,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:07,284 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:12:07,284 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:12:07,553 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:09,554 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:11,555 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:13,556 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:22,428 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:12:22,428 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:12:24,780 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:12:24,780 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:12:24,780 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:12:25,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:12:25,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:27,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:29,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:31,563 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:33,171 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:12:33,564 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:37,559 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:12:37,559 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:12:44,821 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:12:44,822 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:12:44,822 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:12:45,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:12:45,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:47,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:49,571 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:51,572 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:12:52,689 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:12:52,690 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:12:53,573 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:03,249 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:13:04,894 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:13:04,895 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:13:04,895 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:13:05,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:13:05,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:07,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:07,853 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:13:07,854 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:13:09,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:11,580 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:13,581 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:22,998 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:13:22,999 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:13:24,938 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:13:24,938 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:13:24,939 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:13:25,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:13:25,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:26,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:27,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:29,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:31,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:33,328 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:13:33,588 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:38,136 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:13:38,136 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:13:45,048 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:13:45,048 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:13:45,049 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:13:45,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:13:46,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:47,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:49,594 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:51,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:13:53,271 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:13:53,271 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:13:53,596 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:03,405 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:14:05,132 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:14:05,133 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:14:05,133 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:14:05,601 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:14:06,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:07,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:08,431 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:14:08,432 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:14:09,603 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:11,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:13,605 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:23,618 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:14:23,618 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:14:25,238 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:14:25,239 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:14:25,239 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:14:25,610 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:14:25,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:26,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:28,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:30,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:32,614 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:33,477 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:14:34,615 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:38,752 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:14:38,752 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:14:45,277 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:14:45,277 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:14:45,278 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:14:45,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:14:46,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:48,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:50,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:52,622 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:14:53,891 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:14:53,892 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:14:54,623 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:03,547 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:15:05,374 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:15:05,374 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:15:05,375 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:15:05,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:15:06,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:08,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:09,077 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:15:09,078 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:15:10,630 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:12,631 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:14,632 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:24,208 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:15:24,209 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:15:25,504 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:15:25,504 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:15:25,505 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:15:25,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:15:26,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:28,637 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:30,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:32,639 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:33,616 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:15:34,640 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:39,343 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:15:39,343 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:15:45,603 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:15:45,603 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:15:45,603 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:15:45,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:15:46,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:48,645 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:50,646 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:52,647 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:15:54,475 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:15:54,475 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:15:54,647 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:03,685 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:16:05,639 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:16:05,640 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:16:05,640 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:16:05,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:16:06,653 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:08,653 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:09,635 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:16:09,636 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:16:10,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:12,656 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:14,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:24,771 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:16:24,772 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:16:25,733 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:16:25,733 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:16:25,734 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:16:26,662 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:16:26,662 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:27,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:29,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:31,664 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:33,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:33,755 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:16:35,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:39,909 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:16:39,910 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:16:45,881 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:16:45,881 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:16:45,881 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:16:46,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:16:46,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:47,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:49,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:51,674 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:53,675 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:16:55,039 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:16:55,040 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:16:55,675 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:03,828 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:17:05,960 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:17:05,960 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:17:05,961 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:17:06,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:17:06,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:07,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:09,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:11,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:12,129 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:17:12,130 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:17:13,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:26,016 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:17:26,016 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:17:26,017 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:17:26,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:17:27,277 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:17:27,278 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:17:27,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:29,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:31,691 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:33,692 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:33,903 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:17:42,410 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:17:42,410 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:17:46,145 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:17:46,146 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:17:46,146 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:17:46,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:17:47,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:49,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:51,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:53,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:17:57,539 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:17:57,540 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:18:03,976 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:18:06,260 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:18:06,260 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:18:06,261 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:18:06,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:18:07,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:09,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:11,708 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:12,672 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:18:12,672 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:18:13,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:26,331 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:18:26,332 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:18:26,332 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:18:26,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:18:27,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:27,844 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:18:27,844 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:18:30,717 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:32,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:34,054 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:18:34,719 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:42,994 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:18:42,995 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:18:46,414 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:18:46,415 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:18:46,415 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:18:46,723 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:18:47,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:48,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:50,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:52,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:54,727 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:18:58,125 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:18:58,125 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:19:04,131 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:19:06,462 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:19:06,462 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:19:06,463 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:19:06,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:19:07,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:08,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:10,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:12,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:13,257 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:19:13,257 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:19:14,735 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:26,561 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:19:26,561 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:19:26,561 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:19:26,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:19:27,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:28,452 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:19:28,452 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:19:28,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:30,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:32,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:34,209 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:19:34,744 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:43,658 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:19:43,658 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:19:46,638 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:19:46,638 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:19:46,639 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:19:46,749 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:19:47,749 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:48,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:50,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:52,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:54,753 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:19:58,789 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:19:58,790 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:20:04,287 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:20:06,712 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:20:06,713 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:20:06,713 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:20:06,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:20:07,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:08,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:10,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:12,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:13,925 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:20:13,925 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:20:14,762 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:26,812 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:20:26,812 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:20:26,812 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:20:27,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:20:27,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:28,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:29,123 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:20:29,124 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:20:30,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:32,770 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:34,364 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:20:34,771 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:44,273 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:20:44,274 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:20:46,903 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:20:46,903 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:20:46,904 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:20:47,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:20:47,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:48,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:50,778 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:52,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:54,780 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:20:59,402 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:20:59,403 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:21:04,442 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:21:06,986 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:21:06,986 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:21:06,987 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:21:07,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:21:07,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:08,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:11,787 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:13,789 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:14,535 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:21:14,535 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:21:15,789 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:27,032 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:21:27,033 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:21:27,033 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:21:27,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:21:27,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:29,764 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:21:29,764 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:21:29,796 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:31,797 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:33,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:34,521 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:21:35,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:44,914 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:21:44,914 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:21:47,101 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:21:47,101 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:21:47,102 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:21:47,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:21:48,805 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:49,805 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:51,806 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:53,807 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:21:55,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:00,044 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:22:00,044 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:22:04,598 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:22:07,189 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:22:07,189 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:22:07,190 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:22:07,813 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:22:08,814 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:09,814 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:11,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:13,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:15,175 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:22:15,176 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:22:15,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:27,264 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:22:27,264 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:22:27,265 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:22:27,822 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:22:28,822 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:29,823 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:30,325 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:22:30,325 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:22:31,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:33,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:34,676 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:22:35,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:45,490 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:22:45,490 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:22:47,308 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:22:47,308 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:22:47,309 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:22:47,830 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:22:47,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:48,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:49,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:51,832 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:53,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:22:55,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:00,623 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:23:00,623 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:23:04,753 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:23:07,377 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:23:07,378 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:23:07,378 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:23:07,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:23:07,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:08,839 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:10,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:12,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:14,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:15,758 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:23:15,758 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:23:16,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:27,445 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:23:27,445 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:23:27,446 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:23:27,847 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:23:28,848 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:30,848 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:31,258 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:23:31,258 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:23:32,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:34,830 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:23:34,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:36,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:46,405 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:23:46,405 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:23:47,502 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:23:47,503 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:23:47,503 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:23:47,855 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:23:48,855 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:50,856 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:52,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:54,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:23:56,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:01,536 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:24:01,537 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:24:04,909 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:24:07,547 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:24:07,548 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:24:07,548 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:24:07,862 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:24:08,862 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:10,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:12,864 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:14,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:16,671 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:24:16,671 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:24:16,866 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:27,630 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:24:27,630 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:24:27,631 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:24:27,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:24:28,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:30,871 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:31,888 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:24:31,888 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:24:32,872 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:34,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:34,987 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:24:36,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:47,097 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:24:47,098 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:24:47,681 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:24:47,681 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:24:47,682 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:24:47,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:24:48,878 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:50,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:52,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:54,880 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:24:56,881 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:02,230 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:25:02,230 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:25:05,054 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:25:07,720 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:25:07,721 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:25:07,721 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:25:07,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:25:08,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:10,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:12,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:14,888 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:16,889 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:17,372 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:25:17,373 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:25:27,812 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:25:27,812 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:25:27,813 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:25:27,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:25:28,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:30,894 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:32,525 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:25:32,525 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:25:32,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:34,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:35,123 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:25:36,896 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:47,691 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:25:47,692 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:25:47,940 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:25:47,940 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:25:47,941 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:25:48,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:25:48,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:49,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:51,902 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:53,902 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:55,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:25:57,904 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:02,825 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:26:02,825 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:26:05,194 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:26:08,022 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:26:08,022 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:26:08,023 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:26:08,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:26:08,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:09,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:11,909 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:13,910 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:15,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:17,959 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:26:17,960 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:26:28,065 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:26:28,065 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:26:28,066 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:26:28,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:26:29,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:31,917 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:33,101 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:26:33,102 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:26:33,918 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:35,266 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:26:35,919 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:37,919 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:48,137 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:26:48,137 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:26:48,138 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:26:48,325 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:26:48,325 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:26:48,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:26:49,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:51,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:53,926 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:26:55,927 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:03,490 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:27:03,490 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:27:05,340 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:27:08,202 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:27:08,203 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:27:08,203 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:27:08,931 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:27:09,932 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:11,933 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:13,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:15,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:18,627 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:27:18,627 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:27:28,383 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:27:28,383 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:27:28,384 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:27:28,939 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:27:29,939 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:31,940 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:33,813 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:27:33,813 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:27:33,941 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:35,416 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:27:35,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:48,420 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:27:48,420 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:27:48,420 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:27:48,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:27:49,063 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:27:49,064 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:27:49,947 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:51,948 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:53,949 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:55,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:27:57,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:04,212 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:28:04,212 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:28:05,494 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:28:08,540 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:28:08,541 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:28:08,541 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:28:08,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:28:09,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:11,955 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:13,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:15,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:17,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:19,344 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:28:19,345 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:28:28,622 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:28:28,622 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:28:28,623 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:28:28,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:28:29,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:31,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:33,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:34,477 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:28:34,477 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:28:35,572 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:28:35,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:48,720 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:28:48,721 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:28:48,721 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:28:48,969 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:28:49,711 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:28:49,712 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:28:49,969 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:51,970 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:54,971 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:28:56,972 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:04,943 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:29:04,943 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:29:05,647 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:29:08,802 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:29:08,803 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:29:08,803 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:29:08,977 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:29:09,977 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:10,978 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:12,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:14,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:16,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:20,075 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:29:20,075 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:29:28,859 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:29:28,860 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:29:28,860 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:29:28,984 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:29:29,985 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:30,985 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:32,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:34,987 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:35,207 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:29:35,207 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:29:35,721 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:29:36,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:49,015 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:29:49,016 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:29:49,016 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:29:49,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:29:49,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:50,359 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:29:50,359 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:29:50,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:52,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:54,995 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:29:56,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:05,500 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:30:05,501 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:30:05,796 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:30:09,096 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:30:09,097 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:30:09,097 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:30:10,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:30:10,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:11,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:13,002 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:15,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:17,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:20,641 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:30:20,642 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:30:29,147 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:30:29,147 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:30:29,148 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:30:30,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:30:30,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:31,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:33,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:35,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:35,772 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:30:35,773 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:30:35,904 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:30:37,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:49,236 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:30:49,236 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:30:49,237 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:30:50,016 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:30:50,016 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:50,941 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:30:50,941 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:30:51,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:53,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:55,018 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:30:57,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:05,951 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:31:06,131 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:31:06,132 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:31:09,334 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:31:09,334 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:31:09,335 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:31:10,023 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:31:11,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:13,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:15,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:17,026 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:21,268 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:31:21,269 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:31:29,489 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:31:29,490 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:31:29,490 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:31:30,031 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:31:31,031 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:32,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:34,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:36,028 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:31:36,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:36,445 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:31:36,445 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:31:38,034 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:49,571 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:31:49,572 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:31:49,572 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:31:50,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:31:51,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:51,573 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:31:51,574 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:31:52,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:54,040 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:56,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:31:58,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:06,104 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:32:06,714 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:32:06,715 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:32:09,681 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:32:09,681 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:32:09,681 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:32:10,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:32:11,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:12,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:14,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:16,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:18,050 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:21,843 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:32:21,844 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:32:29,771 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:32:29,771 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:32:29,772 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:32:30,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:32:31,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:32,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:34,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:36,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:36,183 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:32:36,986 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:32:36,986 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:32:38,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:49,832 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:32:49,832 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:32:49,833 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:32:50,062 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:32:51,062 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:52,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:52,170 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:32:52,170 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:32:54,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:56,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:32:58,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:06,267 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:33:07,305 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:33:07,305 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:33:09,988 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:33:09,989 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:33:09,989 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:33:10,070 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:33:11,070 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:12,071 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:14,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:16,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:18,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:22,445 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:33:22,445 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:33:30,076 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:33:30,077 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:33:30,077 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:33:30,078 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:33:31,078 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:32,078 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:34,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:36,080 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:36,346 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:33:37,583 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:33:37,584 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:33:38,081 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:50,145 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:33:50,145 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:33:50,145 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:33:51,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:33:51,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:52,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:52,806 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:33:52,806 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:33:54,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:56,088 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:33:58,089 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:06,423 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:34:07,950 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:34:07,950 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:34:10,195 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:34:10,195 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:34:10,196 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:34:11,094 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:34:11,094 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:12,094 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:14,095 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:16,096 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:18,096 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:23,080 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:34:23,081 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:34:30,231 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:34:30,231 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:34:30,232 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:34:31,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:34:31,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:32,102 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:34,102 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:36,103 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:36,501 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:34:38,104 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:38,212 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:34:38,213 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:34:50,341 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:34:50,341 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:34:50,341 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:34:51,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:34:52,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:53,404 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:34:53,405 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:34:54,110 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:57,111 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:34:59,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:06,577 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:35:08,541 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:35:08,541 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:35:10,447 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:35:10,447 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:35:10,448 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:35:11,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:35:12,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:13,118 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:15,118 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:17,119 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:19,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:23,670 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:35:23,671 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:35:30,534 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:35:30,535 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:35:30,535 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:35:31,124 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:35:32,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:33,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:35,126 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:36,656 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:35:37,127 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:38,857 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:35:38,858 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:35:39,127 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:50,585 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:35:50,586 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:35:50,586 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:35:51,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:35:52,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:53,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:54,004 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:35:54,005 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:35:55,134 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:57,135 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:35:59,135 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:06,735 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:36:09,157 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:36:09,158 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:36:10,676 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:36:10,677 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:36:10,677 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:36:11,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:36:12,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:13,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:15,142 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:17,143 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:19,144 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:24,292 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:36:24,292 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:36:30,741 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:36:30,742 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:36:30,742 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:36:31,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:36:32,149 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:33,149 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:35,150 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:36,812 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:36:37,151 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:39,151 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:39,422 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:36:39,422 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:36:50,831 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:36:50,831 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:36:50,832 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:36:51,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:36:52,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:53,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:54,573 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:36:54,573 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:36:55,158 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:57,159 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:36:59,160 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:06,890 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:37:09,732 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:37:09,733 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:37:10,957 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:37:10,957 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:37:10,958 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:37:11,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:37:12,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:13,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:15,166 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:17,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:19,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:24,863 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:37:24,864 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:37:31,034 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:37:31,034 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:37:31,034 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:37:31,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:37:32,173 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:33,173 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:35,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:36,968 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:37:37,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:39,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:39,994 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:37:39,994 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:37:51,146 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:37:51,147 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:37:51,147 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:37:51,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:37:52,181 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:53,181 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:55,126 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:37:55,126 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:37:55,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:57,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:37:59,184 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:07,046 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:38:10,274 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:38:10,274 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:38:11,266 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:38:11,267 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:38:11,267 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:38:12,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:38:12,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:13,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:15,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:17,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:19,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:25,404 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:38:25,405 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:38:31,359 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:38:31,359 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:38:31,359 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:38:32,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:38:32,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:33,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:36,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:37,123 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:38:38,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:40,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:40,533 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:38:40,534 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:38:51,395 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:38:51,396 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:38:51,396 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:38:52,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:38:52,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:54,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:55,670 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:38:55,670 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:38:56,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:38:58,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:00,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:07,199 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:39:10,844 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:39:10,844 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:39:11,455 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:39:11,455 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:39:11,455 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:39:12,213 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:39:13,214 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:14,214 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:16,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:18,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:20,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:25,976 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:39:25,976 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:39:31,538 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:39:31,539 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:39:31,539 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:39:32,221 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:39:33,222 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:34,222 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:36,223 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:37,278 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:39:38,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:40,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:41,109 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:39:41,109 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:39:51,649 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:39:51,650 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:39:51,650 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:39:52,229 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:39:53,229 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:54,230 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:56,231 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:39:56,301 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:39:56,301 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:39:58,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:00,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:07,349 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:40:11,437 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:40:11,437 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:40:11,759 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:40:11,759 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:40:11,760 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:40:12,238 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:40:13,238 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:14,238 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:16,239 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:18,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:20,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:26,566 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:40:26,566 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:40:31,828 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:40:31,829 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:40:31,829 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:40:32,246 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:40:33,246 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:34,247 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:36,247 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:37,419 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:40:38,248 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:40,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:41,700 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:40:41,700 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:40:51,879 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:40:51,879 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:40:51,879 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:40:52,253 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:40:53,254 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:54,254 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:56,255 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:40:56,840 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:40:56,840 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:40:58,256 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:00,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:07,491 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:41:11,950 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:41:11,950 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:41:11,951 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:41:11,989 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:41:11,989 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:41:12,261 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:41:13,261 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:14,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:16,263 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:18,264 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:20,264 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:27,198 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:41:27,198 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:41:31,990 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:41:31,990 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:41:31,990 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:41:32,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:41:33,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:34,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:36,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:37,564 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:41:38,271 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:40,272 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:42,329 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:41:42,329 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:41:52,094 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:41:52,094 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:41:52,094 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:41:52,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:41:53,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:55,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:57,279 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:41:57,502 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:41:57,503 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:41:59,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:01,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:07,640 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:42:12,179 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:42:12,179 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:42:12,180 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:42:12,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:42:12,671 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:42:12,671 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:42:13,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:15,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:17,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:19,288 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:21,289 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:27,800 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:42:27,800 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:42:32,254 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:42:32,254 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:42:32,255 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:42:32,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:42:33,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:35,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:37,295 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:37,719 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:42:39,296 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:41,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:42,936 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:42:42,936 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:42:52,375 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:42:52,376 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:42:52,376 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:42:53,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:42:53,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:55,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:57,303 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:42:58,072 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:42:58,072 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:42:59,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:01,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:07,798 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:43:12,471 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:43:12,472 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:43:12,472 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:43:13,210 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:43:13,210 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:43:13,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:43:13,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:15,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:17,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:19,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:21,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:28,456 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:43:28,457 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:43:32,578 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:43:32,578 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:43:32,579 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:43:33,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:43:33,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:35,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:37,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:37,879 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:43:39,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:41,320 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:43,586 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:43:43,587 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:43:52,636 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:43:52,636 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:43:52,637 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:43:53,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:43:54,325 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:56,326 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:58,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:43:58,716 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:43:58,717 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:44:00,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:02,329 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:07,958 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:44:12,868 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:44:12,869 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:44:12,869 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:44:13,333 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:44:13,892 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:44:13,892 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:44:14,333 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:16,334 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:18,336 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:20,337 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:22,338 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:29,073 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:44:29,073 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:44:32,904 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:44:32,905 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:44:32,905 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:44:33,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:44:34,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:36,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:38,037 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:44:38,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:40,345 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:42,346 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:44,207 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:44:44,207 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:44:53,005 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:44:53,005 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:44:53,006 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:44:53,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:44:54,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:56,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:58,352 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:44:59,335 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:44:59,336 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:45:00,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:02,354 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:08,115 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:45:13,099 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:45:13,099 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:45:13,100 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:45:13,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:45:14,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:14,475 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:45:14,475 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:45:16,361 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:18,362 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:20,362 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:22,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:29,665 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:45:29,665 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:45:33,209 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:45:33,209 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:45:33,210 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:45:33,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:45:34,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:36,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:38,195 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:45:38,370 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:40,371 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:42,372 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:44,798 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:45:44,798 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:45:53,287 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:45:53,287 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:45:53,288 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:45:53,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:45:54,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:56,378 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:58,379 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:45:59,937 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:45:59,938 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:46:00,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:02,381 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:08,273 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:46:13,353 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:46:13,354 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:46:13,354 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:46:13,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:46:14,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:15,117 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:46:15,117 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:46:16,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:18,388 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:20,389 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:22,390 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:30,251 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:46:30,251 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:46:33,439 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:46:33,439 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:46:33,440 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:46:34,395 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:46:34,395 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:36,396 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:38,352 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:46:39,397 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:41,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:43,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:45,385 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:46:45,385 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:46:53,506 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:46:53,507 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:46:53,507 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:46:54,404 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:46:54,405 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:55,405 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:57,406 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:46:59,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:00,574 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:47:00,574 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:47:01,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:03,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:08,430 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:47:13,585 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:47:13,585 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:47:13,586 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:47:14,413 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:47:14,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:15,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:15,747 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:47:15,748 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:47:17,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:19,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:21,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:23,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:30,908 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:47:30,909 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:47:33,664 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:47:33,665 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:47:33,665 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:47:34,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:47:34,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:35,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:37,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:38,507 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:47:39,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:41,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:46,045 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:47:46,045 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:47:53,772 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:47:53,772 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:47:53,772 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:47:54,430 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:47:54,431 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:55,431 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:57,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:47:59,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:01,180 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:48:01,180 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:48:01,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:08,584 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:48:13,834 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:48:13,835 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:48:13,835 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:48:14,439 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:48:14,439 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:15,439 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:16,330 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:48:16,330 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:48:17,440 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:19,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:21,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:23,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:31,467 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:48:31,467 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:48:33,952 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:48:33,952 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:48:33,952 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:48:34,447 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:48:35,447 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:37,448 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:38,662 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:48:39,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:41,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:43,450 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:46,598 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:48:46,599 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:48:54,000 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:48:54,000 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:48:54,000 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:48:54,455 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:48:55,455 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:57,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:48:59,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:01,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:01,733 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:49:01,733 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:49:08,740 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:49:14,064 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:49:14,065 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:49:14,065 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:49:14,462 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:49:15,462 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:16,916 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:49:16,916 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:49:17,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:19,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:21,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:32,048 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:49:32,049 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:49:34,098 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:49:34,098 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:49:34,098 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:49:34,469 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:49:35,469 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:37,470 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:38,818 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:49:39,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:41,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:43,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:47,178 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:49:47,179 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:49:54,143 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:49:54,143 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:49:54,143 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:49:54,477 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:49:55,477 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:57,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:49:59,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:01,479 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:02,310 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:50:02,310 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:50:03,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:08,895 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:50:14,215 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:50:14,216 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:50:14,216 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:50:14,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:50:15,485 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:17,457 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:50:17,457 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:50:17,486 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:19,487 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:21,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:32,618 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:50:32,618 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:50:34,276 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:50:34,277 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:50:34,277 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:50:34,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:50:35,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:36,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:38,495 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:38,973 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:50:40,496 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:42,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:47,749 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:50:47,749 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:50:54,377 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:50:54,377 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:50:54,378 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:50:54,502 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:50:55,503 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:56,503 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:50:58,504 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:00,505 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:02,506 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:02,881 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:51:02,881 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:51:09,051 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:51:14,451 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:51:14,452 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:51:14,452 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:51:14,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:51:15,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:16,512 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:18,014 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:51:18,015 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:51:18,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:20,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:22,515 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:33,197 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:51:33,198 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:51:34,515 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:51:34,516 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:51:34,516 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:51:34,519 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:51:35,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:36,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:38,521 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:39,130 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:51:40,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:42,523 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:48,330 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:51:48,331 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:51:54,621 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:51:54,622 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:51:54,622 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:51:55,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:51:55,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:56,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:51:58,529 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:00,530 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:02,531 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:03,463 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:52:03,463 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:52:09,207 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:52:14,659 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:52:14,660 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:52:14,660 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:52:15,535 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:52:15,535 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:16,536 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:18,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:18,599 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:52:18,599 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:52:20,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:22,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:33,796 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:52:33,796 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:52:34,713 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:52:34,713 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:52:34,713 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:52:35,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:52:35,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:36,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:38,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:39,284 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:52:40,545 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:42,546 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:48,926 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:52:48,927 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:52:54,757 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:52:54,757 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:52:54,758 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:52:55,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:52:56,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:57,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:52:59,553 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:01,553 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:03,554 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:04,062 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:53:04,062 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:53:09,362 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:53:14,862 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:53:14,862 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:53:14,862 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:53:15,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:53:16,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:17,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:19,192 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:53:19,193 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:53:19,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:21,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:23,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:34,332 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:53:34,333 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:53:34,925 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:53:34,926 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:53:34,926 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:53:35,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:53:36,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:37,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:39,440 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:53:39,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:41,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:43,571 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:49,463 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:53:49,464 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:53:55,026 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:53:55,026 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:53:55,027 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:53:55,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:53:56,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:57,577 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:53:59,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:01,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:03,580 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:04,594 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:54:04,594 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:54:09,519 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:54:15,101 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:54:15,101 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:54:15,102 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:54:15,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:54:16,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:17,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:19,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:19,725 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:54:19,725 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:54:21,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:23,588 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:34,861 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:54:34,861 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:54:35,168 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:54:35,169 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:54:35,169 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:54:35,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:54:36,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:37,594 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:39,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:39,596 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:54:41,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:43,596 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:49,998 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:54:49,998 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:54:55,206 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:54:55,207 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:54:55,207 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:54:55,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:54:56,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:57,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:54:59,603 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:01,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:03,605 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:05,135 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:55:05,135 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:55:09,674 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:55:15,278 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:55:15,278 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:55:15,279 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:55:15,610 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:55:16,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:17,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:19,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:20,278 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:55:20,278 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:55:21,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:23,614 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:35,374 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:55:35,374 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:55:35,375 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:55:35,559 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:55:35,559 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:55:35,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:55:36,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:37,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:39,622 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:39,751 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:55:41,623 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:43,624 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:50,766 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:55:50,766 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:55:55,436 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:55:55,436 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:55:55,437 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:55:55,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:55:56,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:57,630 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:55:59,631 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:01,632 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:03,633 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:05,895 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:56:05,895 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:56:09,829 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:56:15,535 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:56:15,536 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:56:15,536 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:56:15,637 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:56:16,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:17,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:19,639 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:21,026 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:56:21,026 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:56:21,640 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:23,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:35,596 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:56:35,597 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:56:35,597 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:56:35,646 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:56:36,246 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:56:36,247 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:56:36,646 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:37,647 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:39,648 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:39,906 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:56:41,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:43,650 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:51,434 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:56:51,435 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:56:55,670 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:56:55,670 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:56:55,671 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:56:56,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:56:56,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:57,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:56:59,656 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:01,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:03,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:06,565 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:57:06,565 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:57:09,984 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:57:15,786 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:57:15,786 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:57:15,787 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:57:16,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:57:16,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:17,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:20,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:21,696 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:57:21,696 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:57:22,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:24,667 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:35,878 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:57:35,878 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:57:35,879 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:57:36,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:57:36,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:36,885 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:57:36,885 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:57:38,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:40,064 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:57:40,674 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:42,675 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:44,676 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:52,044 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:57:52,044 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:57:55,920 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:57:55,921 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:57:55,921 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:57:56,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:57:57,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:57:58,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:00,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:02,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:04,684 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:07,174 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:58:07,175 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:58:10,144 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:58:16,038 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:58:16,039 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:58:16,039 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:58:16,688 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:58:17,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:18,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:20,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:22,307 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:58:22,308 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:58:22,691 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:24,692 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:36,146 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:58:36,146 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:58:36,147 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:58:36,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:58:37,627 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:58:37,627 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:58:37,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:38,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:40,222 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:58:40,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:42,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:44,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:52,764 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:58:52,764 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:58:56,202 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:58:56,203 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:58:56,203 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:58:56,704 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:58:57,705 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:58:58,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:00,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:02,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:04,708 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:07,906 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:59:07,906 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:59:10,299 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:59:16,315 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:59:16,315 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:59:16,316 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:59:16,713 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:59:17,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:18,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:20,716 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:22,717 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:23,037 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:59:23,038 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:59:24,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:36,363 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:59:36,363 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:59:36,364 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:59:36,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:59:37,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:38,267 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:59:38,267 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:59:38,723 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:40,377 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 09:59:40,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:42,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:44,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:53,399 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 09:59:53,399 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 09:59:56,499 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 09:59:56,499 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 09:59:56,500 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 09:59:56,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 09:59:57,731 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 09:59:58,731 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:00,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:02,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:04,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:08,529 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:00:08,529 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:00:10,453 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:00:16,558 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:00:16,559 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:00:16,559 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:00:16,738 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:00:17,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:18,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:20,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:22,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:23,715 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:00:23,715 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:00:24,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:36,623 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:00:36,623 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:00:36,624 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:00:36,748 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:00:38,749 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:38,863 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:00:38,864 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:00:40,522 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:00:40,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:42,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:44,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:00:54,053 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:00:54,053 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:00:56,669 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:00:56,670 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:00:56,670 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:00:56,757 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:00:58,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:00,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:02,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:04,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:09,186 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:01:09,187 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:01:10,590 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:01:16,782 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:01:16,783 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:01:16,783 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:01:17,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:01:18,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:20,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:22,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:24,341 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:01:24,342 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:01:24,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:36,833 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:01:36,833 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:01:36,834 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:01:37,774 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:01:38,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:39,473 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:01:39,474 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:01:40,661 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:01:40,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:43,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:45,778 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:54,631 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:01:54,631 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:01:57,011 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:01:57,012 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:01:57,012 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:01:57,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:01:58,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:01:59,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:01,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:03,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:05,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:09,763 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:02:09,763 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:02:10,732 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:02:17,144 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:02:17,144 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:02:17,144 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:02:17,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:02:18,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:19,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:21,793 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:23,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:24,893 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:02:24,894 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:02:25,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:37,197 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:02:37,198 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:02:37,198 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:02:37,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:02:38,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:39,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:40,033 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:02:40,034 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:02:40,804 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:02:41,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:43,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:45,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:55,184 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:02:55,184 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:02:57,272 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:02:57,273 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:02:57,273 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:02:57,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:02:58,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:02:59,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:01,810 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:03,811 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:05,812 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:10,323 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:03:10,323 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:03:10,879 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:03:17,368 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:03:17,368 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:03:17,368 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:03:17,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:03:18,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:19,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:21,818 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:23,818 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:25,453 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:03:25,453 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:03:25,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:37,456 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:03:37,456 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:03:37,457 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:03:37,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:03:38,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:39,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:40,583 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:03:40,583 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:03:40,954 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:03:41,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:43,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:45,446 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:03:45,446 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:03:45,447 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:03:45,827 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:03:52,830 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:54,830 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:03:55,759 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:03:55,760 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:03:56,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log