diff --git "a/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" "b/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" --- "a/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" +++ "b/wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log" @@ -15750,3 +15750,7325 @@ 2021-07-15 10:03:55,759 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status 2021-07-15 10:03:55,760 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status 2021-07-15 10:03:56,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:04:11,031 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:04:11,146 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:04:11,146 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:04:26,277 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:04:26,277 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:04:41,107 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:04:41,410 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:04:41,410 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:04:56,542 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:04:56,542 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:05:10,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:11,180 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:05:11,742 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:05:11,742 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:05:12,860 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:14,861 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:16,862 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:18,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:26,870 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:05:26,871 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:05:29,653 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:05:29,653 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:05:29,653 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:05:29,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:05:30,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:31,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:32,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:34,869 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:36,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:38,871 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:41,255 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:05:42,001 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:05:42,001 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:05:49,813 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:05:49,814 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:05:49,814 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:05:49,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:05:50,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:51,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:52,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:54,878 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:56,878 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:05:57,131 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:05:57,132 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:05:58,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:09,875 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:06:09,876 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:06:09,876 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:06:09,883 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:06:10,883 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:11,329 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:06:11,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:12,276 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:06:12,276 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:06:12,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:14,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:16,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:18,886 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:27,406 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:06:27,406 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:06:29,944 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:06:29,944 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:06:29,944 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:06:30,891 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:06:30,892 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:31,892 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:33,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:35,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:37,894 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:39,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:41,405 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:06:42,538 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:06:42,539 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:06:50,000 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:06:50,001 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:06:50,001 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:06:50,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:06:51,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:53,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:55,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:57,672 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:06:57,673 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:06:57,902 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:06:59,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:10,097 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:07:10,098 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:07:10,098 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:07:10,907 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:07:11,483 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:07:11,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:12,805 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:07:12,806 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:07:13,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:15,909 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:17,910 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:19,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:27,997 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:07:27,998 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:07:30,205 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:07:30,205 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:07:30,206 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:07:30,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:07:31,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:33,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:35,917 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:37,918 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:41,558 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:07:43,133 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:07:43,134 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:07:50,277 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:07:50,278 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:07:50,279 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:07:50,923 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:07:51,923 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:53,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:55,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:57,926 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:07:58,266 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:07:58,266 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:07:59,927 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:10,335 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:08:10,336 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:08:10,336 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:08:10,932 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:08:11,634 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:08:11,932 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:13,427 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:08:13,428 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:08:13,933 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:15,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:17,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:19,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:28,563 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:08:28,563 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:08:30,457 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:08:30,457 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:08:30,458 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:08:30,940 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:08:31,940 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:33,941 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:35,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:37,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:39,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:41,712 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:08:43,696 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:08:43,696 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:08:50,621 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:08:50,622 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:08:50,622 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:08:50,947 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:08:51,947 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:53,948 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:55,949 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:57,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:08:58,827 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:08:58,828 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:08:59,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:10,629 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:09:10,629 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:09:10,630 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:09:10,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:09:11,809 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:09:12,955 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:14,013 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:09:14,013 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:09:14,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:16,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:18,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:29,168 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:09:29,168 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:09:30,706 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:09:30,706 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:09:30,706 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:09:30,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:09:32,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:34,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:36,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:38,966 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:41,890 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:09:44,311 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:09:44,312 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:09:50,789 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:09:50,789 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:09:50,790 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:09:50,970 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:09:52,971 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:54,972 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:56,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:58,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:09:59,443 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:09:59,443 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:10:10,858 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:10:10,858 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:10:10,858 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:10:10,978 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:10:11,969 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:10:12,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:14,609 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:10:14,609 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:10:14,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:16,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:18,981 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:29,810 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:10:29,810 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:10:30,909 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:10:30,909 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:10:30,909 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:10:30,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:10:32,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:34,987 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:36,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:38,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:42,047 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:10:44,943 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:10:44,943 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:10:50,988 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:10:50,989 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:10:50,989 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:10:50,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:10:52,995 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:54,995 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:56,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:10:58,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:00,074 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:11:00,074 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:11:11,088 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:11:11,088 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:11:11,089 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:11:12,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:11:12,125 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:11:13,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:15,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:15,239 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:11:15,240 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:11:17,005 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:19,005 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:30,418 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:11:30,418 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:11:31,174 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:11:31,175 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:11:31,175 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:11:32,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:11:33,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:35,012 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:37,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:39,014 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:42,202 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:11:45,549 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:11:45,549 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:11:51,258 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:11:51,259 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:11:51,259 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:11:52,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:11:53,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:55,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:57,021 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:11:59,022 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:00,680 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:12:00,680 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:12:11,316 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:12:11,317 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:12:11,317 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:12:12,026 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:12:12,280 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:12:13,027 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:15,028 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:15,817 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:12:15,817 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:12:17,028 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:19,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:30,963 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:12:30,964 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:12:31,463 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:12:31,463 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:12:31,463 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:12:32,034 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:12:33,034 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:35,035 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:37,036 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:39,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:42,358 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:12:46,094 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:12:46,094 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:12:51,510 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:12:51,510 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:12:51,510 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:12:52,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:12:53,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:54,043 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:56,043 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:12:58,044 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:00,045 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:01,227 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:13:01,227 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:13:11,575 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:13:11,575 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:13:11,575 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:13:12,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:13:12,436 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:13:13,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:14,050 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:16,051 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:16,356 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:13:16,356 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:13:18,052 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:20,053 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:31,499 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:13:31,499 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:13:31,654 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:13:31,654 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:13:31,654 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:13:32,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:13:33,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:34,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:36,059 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:38,060 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:40,061 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:42,515 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:13:46,630 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:13:46,631 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:13:51,699 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:13:51,700 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:13:51,700 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:13:52,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:13:53,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:54,067 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:56,068 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:13:58,069 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:00,069 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:01,762 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:14:01,763 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:14:11,775 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:14:11,775 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:14:11,776 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:14:12,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:14:12,591 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:14:14,076 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:16,076 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:16,895 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:14:16,896 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:14:18,077 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:20,078 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:31,845 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:14:31,846 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:14:31,846 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:14:32,043 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:14:32,044 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:14:32,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:14:34,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:36,084 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:38,085 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:40,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:42,669 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:14:47,207 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:14:47,208 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:14:51,884 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:14:51,885 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:14:51,885 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:14:52,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:14:54,091 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:56,092 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:14:58,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:00,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:02,342 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:15:02,343 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:15:11,951 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:15:11,952 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:15:11,952 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:15:12,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:15:12,746 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:15:14,099 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:16,100 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:17,473 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:15:17,474 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:15:18,100 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:20,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:32,070 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:15:32,071 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:15:32,071 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:15:32,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:15:32,609 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:15:32,610 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:15:34,107 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:36,107 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:38,108 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:40,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:42,824 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:15:47,748 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:15:47,748 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:15:52,121 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:15:52,121 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:15:52,121 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:15:53,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:15:54,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:55,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:57,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:15:59,116 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:01,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:02,886 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:16:02,887 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:16:12,176 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:16:12,176 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:16:12,177 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:16:12,897 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:16:13,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:16:14,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:15,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:17,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:18,017 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:16:18,017 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:16:19,124 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:21,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:32,241 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:16:32,242 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:16:32,242 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:16:33,129 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:16:33,198 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:16:33,199 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:16:34,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:35,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:37,131 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:39,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:41,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:42,970 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:16:48,351 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:16:48,351 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:16:52,319 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:16:52,319 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:16:52,320 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:16:53,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:16:53,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:54,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:55,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:57,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:16:59,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:01,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:03,481 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:17:03,481 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:17:12,367 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:17:12,368 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:17:12,368 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:17:13,039 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:17:13,144 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:17:13,145 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:14,145 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:15,145 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:17,146 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:18,613 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:17:18,613 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:17:19,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:21,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:32,421 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:17:32,421 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:17:32,422 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:17:33,152 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:17:33,788 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:17:33,788 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:17:34,153 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:35,153 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:37,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:39,155 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:41,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:43,115 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:17:48,961 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:17:48,961 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:17:52,466 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:17:52,467 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:17:52,467 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:17:53,160 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:17:53,161 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:54,161 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:55,161 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:57,162 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:17:59,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:01,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:04,091 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:18:04,092 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:18:12,544 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:18:12,544 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:18:12,545 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:18:13,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:18:13,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:13,187 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:18:14,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:15,169 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:17,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:19,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:19,223 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:18:19,224 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:18:21,171 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:32,595 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:18:32,595 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:18:32,595 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:18:33,176 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:18:33,176 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:34,176 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:34,353 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:18:34,354 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:18:35,177 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:37,178 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:39,178 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:41,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:43,264 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:18:49,534 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:18:49,534 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:18:52,689 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:18:52,689 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:18:52,690 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:18:53,184 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:18:53,184 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:54,184 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:56,185 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:18:58,186 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:00,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:02,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:04,668 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:19:04,669 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:19:12,752 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:19:12,752 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:19:12,753 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:19:13,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:19:13,338 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:19:14,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:15,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:16,193 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:18,194 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:19,801 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:19:19,801 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:19:20,195 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:22,196 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:32,842 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:19:32,843 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:19:32,843 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:19:33,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:19:34,201 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:34,985 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:19:34,986 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:19:35,201 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:36,202 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:38,203 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:40,203 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:42,204 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:43,416 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:19:50,173 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:19:50,173 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:19:52,923 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:19:52,923 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:19:52,924 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:19:53,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:19:54,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:55,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:56,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:19:58,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:00,211 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:02,212 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:05,309 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:20:05,309 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:20:12,996 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:20:12,996 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:20:12,997 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:20:13,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:20:13,493 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:20:14,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:15,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:16,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:18,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:20,219 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:20,498 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:20:20,498 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:20:22,220 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:33,074 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:20:33,075 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:20:33,075 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:20:33,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:20:34,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:35,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:35,636 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:20:35,636 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:20:36,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:38,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:40,227 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:42,228 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:43,569 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:20:50,806 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:20:50,806 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:20:53,235 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:20:53,235 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:20:53,236 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:20:54,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:20:54,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:55,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:56,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:20:58,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:00,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:03,237 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:05,940 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:21:05,940 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:21:13,369 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:21:13,370 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:21:13,370 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:21:13,647 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:21:14,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:21:15,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:17,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:19,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:21,072 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:21:21,073 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:21:21,244 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:33,429 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:21:33,429 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:21:33,430 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:21:34,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:21:35,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:36,211 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:21:36,212 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:21:37,251 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:39,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:41,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:43,253 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:43,726 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:21:51,349 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:21:51,350 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:21:53,478 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:21:53,478 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:21:53,479 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:21:54,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:21:55,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:57,259 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:21:59,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:01,261 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:03,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:06,489 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:22:06,489 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:22:13,563 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:22:13,564 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:22:13,564 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:22:13,805 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:22:14,267 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:22:15,267 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:17,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:19,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:21,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:21,623 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:22:21,623 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:22:33,629 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:22:33,629 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:22:33,630 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:22:34,275 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:22:35,275 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:36,829 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:22:36,830 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:22:37,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:39,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:41,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:43,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:43,883 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:22:51,973 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:22:51,974 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:22:53,670 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:22:53,670 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:22:53,671 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:22:54,282 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:22:55,283 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:57,283 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:22:59,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:01,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:07,106 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:23:07,106 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:23:13,718 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:23:13,718 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:23:13,719 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:23:13,960 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:23:14,290 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:23:15,290 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:17,291 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:19,291 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:21,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:22,235 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:23:22,235 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:23:33,832 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:23:33,833 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:23:33,833 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:23:34,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:23:35,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:36,297 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:37,298 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:37,368 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:23:37,368 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:23:39,299 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:41,299 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:44,038 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:23:52,568 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:23:52,568 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:23:53,900 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:23:53,900 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:23:53,900 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:23:54,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:23:55,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:56,305 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:57,305 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:23:59,306 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:01,307 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:03,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:07,843 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:24:07,843 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:24:13,981 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:24:13,982 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:24:13,982 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:24:14,116 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:24:14,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:24:15,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:16,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:17,313 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:19,313 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:21,314 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:22,973 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:24:22,973 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:24:23,315 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:34,050 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:24:34,051 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:24:34,051 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:24:34,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:24:35,320 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:36,320 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:38,117 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:24:38,117 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:24:38,321 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:40,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:42,323 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:44,194 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:24:53,249 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:24:53,249 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:24:54,107 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:24:54,107 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:24:54,107 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:24:54,328 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:24:56,329 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:24:58,330 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:00,331 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:02,332 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:08,382 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:25:08,382 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:25:14,171 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:25:14,171 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:25:14,171 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:25:14,270 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:25:14,336 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:25:16,337 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:18,338 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:20,339 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:22,340 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:23,513 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:25:23,513 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:25:34,260 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:25:34,261 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:25:34,261 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:25:34,345 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:25:36,346 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:38,347 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:38,648 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:25:38,648 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:25:40,348 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:42,349 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:44,349 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:25:53,867 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:25:53,868 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:25:54,295 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:25:54,295 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:25:54,296 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:25:54,356 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:25:56,357 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:25:58,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:00,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:02,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:08,999 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:26:08,999 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:26:14,336 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:26:14,337 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:26:14,337 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:26:14,365 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:26:14,426 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:26:16,366 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:18,367 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:20,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:22,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:24,129 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:26:24,129 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:26:34,407 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:26:34,408 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:26:34,408 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:26:35,374 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:26:36,374 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:38,375 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:39,262 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:26:39,263 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:26:40,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:42,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:44,504 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:26:54,393 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:26:54,393 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:26:54,524 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:26:54,525 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:26:54,525 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:26:55,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:26:56,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:26:58,384 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:00,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:02,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:09,537 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:27:09,537 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:27:14,531 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:27:14,531 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:27:14,534 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:27:14,582 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:27:15,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:27:16,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:18,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:20,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:22,394 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:24,668 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:27:24,668 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:27:34,599 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:27:34,600 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:27:34,600 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:27:35,400 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:27:36,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:38,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:39,798 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:27:39,799 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:27:40,403 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:42,404 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:44,660 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:27:54,668 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:27:54,668 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:27:54,669 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:27:54,935 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:27:54,936 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:27:55,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:27:56,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:27:58,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:00,411 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:02,411 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:10,094 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:28:10,094 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:28:14,723 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:28:14,723 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:28:14,725 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:28:14,738 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:28:15,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:28:16,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:17,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:18,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:20,419 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:22,420 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:25,226 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:28:25,227 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:28:34,798 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:28:34,798 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:28:34,798 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:28:35,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:28:36,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:37,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:39,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:40,356 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:28:40,356 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:28:41,429 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:43,429 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:44,816 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:28:54,870 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:28:54,871 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:28:54,871 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:28:55,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:28:55,513 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:28:55,513 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:28:57,435 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:28:59,436 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:01,437 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:03,438 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:10,680 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:29:10,680 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:29:14,895 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:29:14,974 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:29:14,975 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:29:14,975 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:29:15,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:29:17,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:19,444 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:21,445 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:23,445 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:25,809 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:29:25,809 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:29:35,026 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:29:35,026 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:29:35,026 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:29:35,450 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:29:37,451 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:39,452 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:40,939 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:29:40,940 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:29:41,452 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:43,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:44,973 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:29:55,115 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:29:55,116 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:29:55,116 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:29:55,458 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:29:56,075 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:29:56,075 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:29:57,459 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:29:59,460 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:01,461 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:03,462 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:11,230 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:30:11,230 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:30:15,053 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:30:15,148 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:30:15,148 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:30:15,148 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:30:15,467 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:30:17,468 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:19,469 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:21,470 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:23,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:26,363 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:30:26,364 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:30:35,224 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:30:35,225 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:30:35,225 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:30:35,476 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:30:37,477 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:39,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:41,479 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:41,494 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:30:41,494 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:30:43,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:45,131 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:30:55,299 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:30:55,299 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:30:55,299 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:30:55,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:30:56,670 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:30:56,670 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:30:57,485 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:30:59,486 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:01,487 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:03,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:11,821 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:31:11,822 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:31:15,209 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:31:15,370 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:31:15,370 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:31:15,370 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:31:15,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:31:17,495 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:19,496 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:21,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:23,498 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:26,951 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:31:26,951 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:31:35,448 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:31:35,449 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:31:35,449 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:31:35,504 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:31:37,504 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:39,505 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:41,506 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:42,087 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:31:42,088 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:31:43,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:45,287 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:31:55,510 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:31:55,510 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:31:55,511 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:31:55,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:31:57,224 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:31:57,224 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:31:57,512 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:31:59,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:01,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:03,515 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:12,421 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:32:12,421 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:32:15,364 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:32:15,579 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:32:15,579 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:32:15,580 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:32:16,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:32:18,521 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:20,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:22,523 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:24,524 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:27,552 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:32:27,553 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:32:35,663 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:32:35,663 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:32:35,664 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:32:36,529 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:32:38,530 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:40,531 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:42,532 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:42,683 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:32:42,683 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:32:44,533 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:32:45,443 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:32:55,776 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:32:55,777 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:32:55,777 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:32:56,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:32:57,820 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:32:57,821 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:32:58,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:00,539 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:02,540 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:04,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:12,995 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:33:12,995 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:33:15,519 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:33:15,879 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:33:15,880 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:33:15,880 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:33:16,547 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:33:18,547 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:20,548 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:22,549 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:24,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:28,126 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:33:28,126 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:33:35,971 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:33:35,971 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:33:35,972 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:33:36,555 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:33:38,556 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:40,557 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:42,558 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:43,257 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:33:43,257 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:33:44,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:33:45,597 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:33:56,033 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:33:56,033 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:33:56,034 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:33:56,564 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:33:58,455 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:33:58,455 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:33:58,565 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:00,565 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:02,566 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:04,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:13,596 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:34:13,596 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:34:15,675 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:34:16,093 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:34:16,093 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:34:16,094 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:34:16,573 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:34:18,573 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:20,574 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:22,575 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:24,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:28,732 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:34:28,733 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:34:36,228 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:34:36,228 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:34:36,228 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:34:36,581 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:34:38,582 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:40,583 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:42,583 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:43,863 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:34:43,863 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:34:44,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:45,754 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:34:56,272 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:34:56,272 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:34:56,273 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:34:56,589 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:34:58,589 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:34:58,999 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:34:58,999 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:35:00,590 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:02,591 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:04,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:14,250 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:35:14,251 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:35:15,834 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:35:16,358 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:35:16,358 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:35:16,359 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:35:16,597 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:35:18,598 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:20,599 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:22,599 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:24,600 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:29,387 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:35:29,387 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:35:36,406 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:35:36,407 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:35:36,407 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:35:36,605 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:35:38,606 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:40,606 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:42,607 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:44,526 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:35:44,526 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:35:44,608 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:45,913 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:35:56,522 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:35:56,522 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:35:56,523 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:35:56,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:35:58,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:35:59,739 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:35:59,740 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:36:01,614 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:03,615 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:05,616 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:14,891 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:36:14,891 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:36:15,992 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:36:16,662 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:36:16,663 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:36:16,663 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:36:17,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:36:18,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:19,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:21,622 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:23,623 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:25,623 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:30,023 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:36:30,023 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:36:36,735 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:36:36,735 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:36:36,736 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:36:37,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:36:38,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:39,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:41,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:43,630 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:45,153 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:36:45,154 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:36:45,631 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:46,069 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:36:56,808 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:36:56,808 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:36:56,809 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:36:57,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:36:58,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:36:59,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:00,306 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:37:00,307 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:37:01,637 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:03,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:05,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:15,490 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:37:15,491 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:37:16,145 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:37:16,896 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:37:16,896 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:37:16,897 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:37:17,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:37:17,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:18,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:19,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:21,645 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:23,646 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:25,647 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:30,621 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:37:30,622 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:37:36,949 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:37:36,949 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:37:36,950 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:37:37,651 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:37:37,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:39,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:41,653 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:43,654 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:45,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:37:45,752 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:37:45,752 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:37:46,223 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:37:57,014 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:37:57,014 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:37:57,015 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:37:57,660 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:37:59,661 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:00,932 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:38:00,932 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:38:01,661 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:03,662 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:05,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:16,149 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:38:16,150 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:38:16,292 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:38:17,063 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:38:17,063 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:38:17,064 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:38:17,668 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:38:19,669 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:21,670 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:23,670 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:25,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:31,280 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:38:31,280 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:38:37,175 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:38:37,176 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:38:37,176 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:38:37,676 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:38:37,676 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:39,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:41,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:44,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:46,360 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:38:46,409 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:38:46,409 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:38:46,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:57,278 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:38:57,278 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:38:57,279 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:38:57,684 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:38:58,685 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:38:59,685 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:00,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:01,559 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:39:01,560 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:39:02,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:04,687 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:06,688 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:16,432 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:39:16,698 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:39:16,699 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:39:17,352 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:39:17,352 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:39:17,352 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:39:17,692 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:39:18,693 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:19,693 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:20,694 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:22,694 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:24,695 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:26,696 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:31,829 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:39:31,829 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:39:37,410 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:39:37,411 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:39:37,411 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:39:37,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:39:38,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:39,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:40,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:42,703 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:44,704 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:46,506 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:39:46,705 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:46,960 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:39:46,961 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:39:57,458 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:39:57,459 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:39:57,459 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:39:57,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:39:58,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:39:59,711 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:00,711 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:02,141 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:40:02,141 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:40:02,712 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:04,713 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:06,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:16,578 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:40:17,286 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:40:17,287 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:40:17,493 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:40:17,493 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:40:17,494 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:40:17,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:40:18,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:19,719 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:20,719 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:22,720 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:24,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:26,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:32,416 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:40:32,417 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:40:37,600 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:40:37,600 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:40:37,601 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:40:37,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:40:38,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:39,727 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:40,727 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:42,728 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:44,729 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:46,653 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:40:46,729 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:47,553 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:40:47,553 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:40:57,658 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:40:57,659 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:40:57,659 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:40:57,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:40:58,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:40:59,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:00,735 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:02,684 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:41:02,684 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:41:02,736 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:04,737 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:06,738 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:16,729 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:41:17,703 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:41:17,704 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:41:17,704 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:41:17,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:41:17,829 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:41:17,830 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:41:18,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:19,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:20,744 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:22,745 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:24,745 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:26,746 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:32,964 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:41:32,964 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:41:37,756 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:41:37,756 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:41:37,757 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:41:38,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:41:38,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:39,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:40,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:42,753 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:44,754 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:46,755 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:41:46,805 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:41:48,097 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:41:48,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:41:57,885 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:41:57,885 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:41:57,886 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:41:58,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:41:58,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:00,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:02,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:03,229 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:42:03,230 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:42:04,762 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:06,763 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:16,884 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:42:18,012 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:42:18,013 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:42:18,013 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:42:18,369 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:42:18,369 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:42:18,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:42:19,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:20,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:21,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:23,770 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:25,771 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:27,772 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:33,506 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:42:33,507 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:42:38,051 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:42:38,052 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:42:38,052 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:42:38,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:42:39,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:40,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:41,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:43,778 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:45,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:46,960 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:42:47,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:42:48,636 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:42:48,637 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:42:58,125 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:42:58,125 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:42:58,126 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:42:58,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:42:59,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:00,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:01,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:03,767 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:43:03,767 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:43:03,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:05,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:07,787 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:17,039 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:43:18,213 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:43:18,213 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:43:18,214 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:43:18,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:43:18,905 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:43:18,905 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:43:19,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:20,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:21,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:23,793 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:25,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:27,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:34,062 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:43:34,063 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:43:38,288 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:43:38,289 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:43:38,289 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:43:38,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:43:39,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:40,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:41,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:43,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:45,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:47,116 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:43:47,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:43:49,194 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:43:49,195 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:43:58,352 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:43:58,352 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:43:58,352 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:43:58,806 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:43:59,807 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:00,807 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:01,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:03,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:04,327 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:44:04,328 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:44:05,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:17,195 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:44:18,388 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:44:18,389 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:44:18,389 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:44:18,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:44:19,533 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:44:19,534 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:44:19,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:20,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:21,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:23,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:25,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:34,665 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:44:34,665 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:44:38,436 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:44:38,436 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:44:38,436 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:44:38,822 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:44:39,823 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:40,823 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:41,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:43,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:45,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:47,272 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:44:47,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:44:49,795 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:44:49,795 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:44:58,548 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:44:58,548 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:44:58,549 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:44:58,830 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:44:59,830 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:00,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:01,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:03,832 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:04,925 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:45:04,925 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:45:05,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:07,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:17,350 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:45:18,647 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:45:18,647 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:45:18,648 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:45:18,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:45:20,072 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:45:20,072 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:45:20,839 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:22,839 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:24,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:26,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:35,205 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:45:35,205 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:45:38,698 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:45:38,699 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:45:38,699 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:45:38,846 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:45:40,847 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:42,848 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:44,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:46,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:45:47,427 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:45:50,336 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:45:50,337 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:45:58,745 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:45:58,745 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:45:58,746 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:45:58,855 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:46:00,856 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:02,856 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:04,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:05,467 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:46:05,467 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:46:06,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:17,505 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:46:18,788 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:46:18,788 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:46:18,789 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:46:18,863 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:46:20,712 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:46:20,712 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:46:20,864 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:22,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:24,866 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:26,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:35,914 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:46:35,915 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:46:38,859 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:46:38,860 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:46:38,860 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:46:38,871 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:46:40,872 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:42,872 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:44,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:46,874 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:46:47,583 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:46:51,047 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:46:51,048 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:46:58,970 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:46:58,971 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:46:58,971 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:46:59,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:47:00,879 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:02,880 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:04,881 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:06,189 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:47:06,189 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:47:06,882 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:17,661 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:47:19,103 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:47:19,104 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:47:19,104 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:47:19,887 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:47:20,888 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:21,394 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:47:21,394 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:47:21,888 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:22,888 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:24,889 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:26,890 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:36,635 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:47:36,635 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:47:39,245 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:47:39,245 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:47:39,246 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:47:39,895 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:47:40,896 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:41,896 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:42,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:44,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:46,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:47:47,739 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:47:51,770 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:47:51,770 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:47:59,291 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:47:59,292 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:47:59,292 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:47:59,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:48:00,904 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:01,904 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:02,905 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:04,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:06,899 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:48:06,899 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:48:06,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:17,816 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:48:19,412 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:48:19,412 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:48:19,412 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:48:19,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:48:20,912 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:21,912 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:22,066 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:48:22,066 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:48:23,913 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:25,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:27,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:37,197 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:48:37,198 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:48:39,508 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:48:39,509 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:48:39,509 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:48:39,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:48:41,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:43,921 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:45,922 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:47,895 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:48:47,923 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:48:52,329 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:48:52,329 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:48:59,563 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:48:59,563 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:48:59,564 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:48:59,927 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:49:01,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:03,929 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:05,930 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:07,461 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:49:07,461 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:49:07,930 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:17,970 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:49:19,640 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:49:19,640 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:49:19,641 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:49:19,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:49:21,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:22,622 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:49:22,623 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:49:23,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:25,938 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:27,939 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:37,763 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:49:37,764 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:49:39,723 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:49:39,723 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:49:39,724 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:49:39,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:49:41,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:43,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:45,945 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:47,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:49:48,039 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:49:52,898 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:49:52,898 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:49:59,792 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:49:59,793 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:49:59,793 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:49:59,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:50:01,951 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:03,952 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:05,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:07,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:08,030 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:50:08,030 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:50:18,107 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:50:19,828 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:50:19,828 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:50:19,829 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:50:19,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:50:21,959 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:23,185 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:50:23,185 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:50:23,960 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:25,961 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:27,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:38,328 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:50:38,329 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:50:39,899 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:50:39,899 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:50:39,899 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:50:39,966 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:50:41,967 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:43,968 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:45,968 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:47,969 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:50:48,184 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:50:53,462 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:50:53,462 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:50:59,940 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:50:59,940 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:50:59,941 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:50:59,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:51:01,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:03,975 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:05,975 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:07,976 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:08,605 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:51:08,606 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:51:18,263 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:51:19,995 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:51:19,996 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:51:19,996 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:51:20,981 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:51:21,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:23,740 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:51:23,740 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:51:23,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:25,983 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:27,984 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:38,894 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:51:38,894 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:51:40,043 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:51:40,044 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:51:40,044 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:51:40,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:51:41,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:43,990 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:46,992 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:48,339 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:51:48,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:51:54,024 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:51:54,024 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:52:00,119 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:52:00,120 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:52:00,120 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:52:00,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:52:01,998 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:02,998 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:04,999 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:07,000 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:09,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:09,155 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:52:09,155 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:52:18,418 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:52:20,155 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:52:20,156 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:52:20,156 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:52:21,006 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:52:23,006 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:24,290 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:52:24,291 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:52:25,007 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:27,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:29,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:39,507 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:52:39,507 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:52:40,303 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:52:40,303 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:52:40,303 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:52:41,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:52:43,014 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:45,015 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:47,016 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:48,496 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:52:49,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:52:54,638 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:52:54,638 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:53:00,357 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:53:00,358 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:53:00,358 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:53:01,022 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:53:03,023 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:05,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:07,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:09,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:09,767 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:53:09,768 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:53:18,573 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:53:20,454 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:53:20,454 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:53:20,454 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:53:21,030 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:53:23,031 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:24,962 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:53:24,962 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:53:25,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:27,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:29,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:40,095 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:53:40,095 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:53:40,512 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:53:40,512 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:53:40,513 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:53:41,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:53:41,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:43,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:45,040 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:47,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:48,651 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:53:49,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:53:55,233 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:53:55,233 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:54:00,564 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:54:00,564 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:54:00,565 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:54:01,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:54:03,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:05,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:07,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:09,050 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:10,415 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:54:10,415 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:54:18,728 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:54:20,615 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:54:20,615 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:54:20,616 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:54:21,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:54:23,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:25,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:25,565 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:54:25,565 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:54:27,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:29,059 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:40,680 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:54:40,681 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:54:40,681 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:54:40,719 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:54:40,719 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:54:41,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:54:41,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:43,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:45,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:47,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:48,805 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:54:49,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:54:55,930 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:54:55,931 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:55:00,714 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:55:00,715 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:55:00,715 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:55:01,071 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:55:03,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:04,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:06,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:08,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:10,075 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:11,060 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:55:11,060 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:55:18,885 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:55:20,776 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:55:20,777 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:55:20,777 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:55:21,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:55:23,080 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:24,081 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:26,082 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:26,200 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:55:26,201 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:55:28,082 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:30,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:40,860 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:55:40,860 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:55:40,861 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:55:41,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:55:41,332 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:55:41,333 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:55:42,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:43,088 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:44,088 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:46,089 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:48,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:48,962 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:55:50,091 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:55:56,468 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:55:56,469 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:56:00,981 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:56:00,981 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:56:00,982 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:56:01,095 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:56:03,096 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:04,097 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:06,097 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:08,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:10,099 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:11,602 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:56:11,602 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:56:19,039 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:56:21,063 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:56:21,064 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:56:21,064 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:56:21,103 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:56:24,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:26,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:26,734 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:56:26,734 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:56:28,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:30,107 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:41,177 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:56:41,178 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:56:41,178 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:56:41,907 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:56:41,908 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:56:42,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:56:44,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:46,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:48,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:49,118 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:56:50,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:56:57,048 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:56:57,049 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:57:01,239 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:57:01,240 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:57:01,240 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:57:02,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:57:02,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:04,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:06,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:08,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:10,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:12,177 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:57:12,178 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:57:19,195 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:57:21,318 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:57:21,319 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:57:21,319 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:57:22,128 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:57:22,128 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:24,128 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:26,129 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:27,309 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:57:27,309 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:57:28,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:30,131 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:41,415 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:57:41,415 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:57:41,416 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:57:42,135 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:57:42,136 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:42,564 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:57:42,564 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:57:44,136 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:46,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:48,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:49,272 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:57:50,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:57:57,719 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:57:57,720 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:58:01,480 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:58:01,481 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:58:01,481 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:58:02,143 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:58:04,144 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:06,144 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:08,145 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:10,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:12,849 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:58:12,850 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:58:19,353 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:58:21,551 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:58:21,552 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:58:21,552 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:58:22,151 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:58:22,151 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:24,152 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:26,153 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:27,981 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:58:27,982 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:58:29,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:31,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:41,602 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:58:41,602 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:58:41,603 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:58:42,158 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:58:43,159 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:43,200 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:58:43,201 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:58:44,159 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:45,160 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:47,161 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:49,162 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:49,434 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:58:51,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:58:58,397 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:58:58,397 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:59:01,722 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:59:01,722 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:59:01,723 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:59:02,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:59:03,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:04,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:05,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:07,169 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:09,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:11,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:13,530 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:59:13,530 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:59:19,511 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:59:21,774 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:59:21,774 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:59:21,775 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:59:22,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:59:23,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:24,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:25,176 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:27,177 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:28,729 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:59:28,730 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:59:29,177 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:31,178 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:41,841 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 10:59:41,842 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 10:59:41,842 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 10:59:42,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 10:59:43,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:43,860 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:59:43,861 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 10:59:44,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:45,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:47,184 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:49,185 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:49,591 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 10:59:51,186 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 10:59:59,021 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 10:59:59,021 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:00:01,970 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:00:01,970 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:00:01,971 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:00:02,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:00:03,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:04,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:05,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:07,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:09,193 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:11,194 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:14,152 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:00:14,153 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:00:19,671 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:00:22,058 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:00:22,059 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:00:22,059 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:00:22,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:00:23,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:24,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:25,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:27,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:29,201 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:29,285 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:00:29,286 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:00:31,202 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:42,136 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:00:42,137 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:00:42,137 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:00:42,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:00:43,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:44,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:44,473 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:00:44,473 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:00:45,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:47,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:49,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:49,750 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:00:51,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:00:59,660 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:00:59,660 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:01:02,186 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:01:02,187 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:01:02,187 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:01:02,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:01:03,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:04,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:05,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:07,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:09,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:11,219 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:14,789 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:01:14,789 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:01:19,827 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:01:22,245 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:01:22,246 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:01:22,246 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:01:23,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:01:23,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:24,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:25,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:27,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:29,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:29,925 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:01:29,926 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:01:31,227 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:42,329 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:01:42,329 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:01:42,330 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:01:43,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:01:43,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:44,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:45,055 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:01:45,055 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:01:45,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:47,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:49,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:01:49,905 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:01:51,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:00,392 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:02:00,392 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:02:02,440 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:02:02,440 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:02:02,441 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:02:03,239 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:02:03,239 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:04,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:05,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:07,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:09,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:11,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:15,522 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:02:15,523 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:02:19,979 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:02:22,517 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:02:22,517 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:02:22,518 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:02:23,247 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:02:24,247 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:25,248 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:26,248 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:28,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:30,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:30,654 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:02:30,654 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:02:32,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:42,641 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:02:42,642 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:02:42,642 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:02:43,254 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:02:44,255 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:45,255 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:45,789 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:02:45,789 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:02:46,256 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:48,256 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:50,050 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:02:50,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:02:52,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:00,958 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:03:00,958 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:03:02,708 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:03:02,708 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:03:02,709 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:03:03,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:03:04,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:05,263 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:06,263 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:08,264 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:10,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:12,265 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:16,088 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:03:16,088 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:03:20,124 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:03:22,744 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:03:22,744 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:03:22,745 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:03:23,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:03:24,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:25,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:26,271 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:28,271 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:30,273 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:31,221 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:03:31,222 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:03:32,273 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:42,808 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:03:42,808 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:03:42,809 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:03:43,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:03:44,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:45,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:46,279 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:46,382 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:03:46,382 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:03:48,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:50,200 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:03:50,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:03:52,281 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:01,551 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:04:01,551 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:04:02,869 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:04:02,870 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:04:02,870 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:04:03,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:04:04,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:05,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:06,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:08,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:10,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:12,288 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:16,682 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:04:16,682 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:04:20,275 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:04:22,999 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:04:23,000 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:04:23,000 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:04:23,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:04:24,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:25,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:26,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:28,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:30,295 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:31,814 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:04:31,814 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:04:32,296 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:43,080 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:04:43,081 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:04:43,081 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:04:43,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:04:44,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:45,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:46,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:46,958 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:04:46,958 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:04:48,303 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:50,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:04:50,350 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:04:52,304 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:02,139 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:05:02,139 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:05:03,187 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:05:03,187 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:05:03,188 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:05:03,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:05:04,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:05,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:06,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:08,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:10,312 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:12,313 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:17,270 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:05:17,270 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:05:20,428 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:05:23,342 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:05:23,343 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:05:23,343 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:05:24,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:05:24,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:25,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:26,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:28,320 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:30,321 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:32,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:32,404 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:05:32,404 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:05:43,397 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:05:43,398 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:05:43,398 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:05:44,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:05:44,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:45,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:46,328 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:47,610 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:05:47,610 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:05:48,329 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:50,330 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:05:50,506 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:05:52,330 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:02,747 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:06:02,747 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:06:03,514 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:06:03,515 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:06:03,515 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:06:04,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:06:04,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:05,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:06,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:08,336 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:10,337 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:12,338 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:17,879 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:06:17,879 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:06:20,584 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:06:23,562 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:06:23,562 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:06:23,563 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:06:24,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:06:25,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:27,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:29,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:31,345 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:33,009 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:06:33,009 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:06:33,346 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:43,647 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:06:43,647 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:06:43,647 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:06:44,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:06:45,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:47,352 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:48,179 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:06:48,179 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:06:49,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:50,661 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:06:51,354 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:06:53,355 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:03,312 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:07:03,313 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:07:03,684 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:07:03,684 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:07:03,685 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:07:04,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:07:05,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:07,361 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:09,362 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:11,363 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:18,465 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:07:18,465 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:07:20,740 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:07:23,759 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:07:23,760 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:07:23,760 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:07:24,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:07:25,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:27,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:29,370 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:31,371 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:33,372 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:33,691 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:07:33,691 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:07:43,831 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:07:43,831 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:07:43,831 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:07:44,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:07:45,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:46,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:47,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:48,826 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:07:48,826 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:07:49,378 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:50,816 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:07:51,379 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:07:53,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:03,866 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:08:03,867 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:08:03,867 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:08:04,004 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:08:04,005 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:08:04,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:08:05,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:06,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:07,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:09,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:11,388 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:19,177 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:08:19,177 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:08:20,895 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:08:23,919 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:08:23,919 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:08:23,919 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:08:24,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:08:25,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:26,394 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:27,394 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:29,395 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:31,396 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:34,307 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:08:34,308 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:08:43,989 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:08:43,989 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:08:43,990 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:08:44,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:08:45,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:46,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:47,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:49,403 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:08:49,438 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:08:49,439 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:08:50,973 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:08:51,404 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:04,031 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:09:04,032 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:09:04,032 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:09:04,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:09:04,607 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:09:04,608 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:09:05,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:06,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:07,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:09,411 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:11,412 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:13,412 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:20,224 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:09:20,224 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:09:21,053 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:09:24,071 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:09:24,071 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:09:24,071 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:09:24,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:09:25,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:26,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:27,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:29,419 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:31,419 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:35,355 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:09:35,355 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:09:44,142 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:09:44,143 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:09:44,143 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:09:44,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:09:45,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:46,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:48,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:50,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:09:50,491 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:09:50,491 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:09:51,130 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:09:52,428 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:04,222 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:10:04,222 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:10:04,222 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:10:04,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:10:05,639 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:10:05,639 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:10:06,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:08,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:10,435 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:12,436 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:20,820 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:10:20,820 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:10:21,207 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:10:24,297 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:10:24,298 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:10:24,298 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:10:24,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:10:26,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:28,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:30,444 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:32,445 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:35,951 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:10:35,952 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:10:44,332 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:10:44,333 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:10:44,333 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:10:44,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:10:46,450 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:48,451 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:50,452 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:10:51,081 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:10:51,081 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:10:51,286 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:10:52,452 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:04,426 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:11:04,427 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:11:04,427 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:11:04,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:11:06,257 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:11:06,258 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:11:06,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:08,458 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:10,459 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:12,460 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:21,367 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:11:21,447 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:11:21,447 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:11:24,490 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:11:24,491 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:11:24,491 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:11:25,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:11:26,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:28,466 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:30,467 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:32,468 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:36,580 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:11:36,580 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:11:44,543 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:11:44,543 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:11:44,544 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:11:45,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:11:46,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:47,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:48,474 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:50,475 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:11:51,445 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:11:51,710 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:11:51,711 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:11:52,476 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:04,588 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:12:04,588 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:12:04,589 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:12:05,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:12:06,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:06,870 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:12:06,870 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:12:07,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:08,482 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:10,483 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:12,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:21,522 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:12:22,035 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:12:22,036 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:12:24,626 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:12:24,627 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:12:24,627 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:12:25,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:12:26,490 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:27,490 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:28,491 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:30,492 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:32,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:37,164 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:12:37,165 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:12:44,662 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:12:44,662 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:12:44,663 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:12:45,498 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:12:47,499 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:49,500 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:51,501 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:12:51,600 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:12:52,295 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:12:52,295 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:12:53,502 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:04,749 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:13:04,750 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:13:04,750 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:13:05,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:13:05,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:07,427 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:13:07,428 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:13:07,508 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:09,509 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:11,510 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:13,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:21,677 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:13:22,583 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:13:22,583 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:13:24,816 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:13:24,816 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:13:24,816 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:13:25,516 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:13:27,517 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:29,518 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:31,519 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:33,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:37,714 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:13:37,716 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:13:44,893 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:13:44,893 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:13:44,894 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:13:45,525 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:13:47,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:49,527 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:51,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:13:51,757 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:13:52,847 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:13:52,847 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:13:53,529 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:04,988 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:14:04,988 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:14:04,989 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:14:05,534 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:14:07,535 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:08,004 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:14:08,004 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:14:09,536 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:11,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:13,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:21,834 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:14:23,161 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:14:23,161 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:14:25,037 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:14:25,038 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:14:25,038 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:14:25,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:14:27,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:29,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:31,545 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:33,545 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:38,293 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:14:38,294 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:14:45,123 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:14:45,123 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:14:45,124 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:14:45,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:14:45,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:47,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:49,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:51,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:14:51,915 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:14:53,425 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:14:53,425 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:14:53,554 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:05,231 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:15:05,232 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:15:05,232 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:15:05,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:15:07,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:08,585 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:15:08,585 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:15:09,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:11,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:13,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:21,992 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:15:23,766 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:15:23,767 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:15:25,358 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:15:25,358 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:15:25,359 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:15:25,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:15:27,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:29,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:31,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:33,571 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:38,897 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:15:38,897 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:15:45,431 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:15:45,432 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:15:45,432 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:15:45,575 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:15:47,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:48,577 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:49,577 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:51,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:52,071 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:15:53,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:15:54,030 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:15:54,030 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:16:05,488 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:16:05,489 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:16:05,489 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:16:05,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:16:07,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:08,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:09,181 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:16:09,181 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:16:09,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:11,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:13,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:22,149 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:16:24,349 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:16:24,350 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:16:25,543 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:16:25,544 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:16:25,544 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:16:25,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:16:26,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:27,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:29,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:32,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:34,596 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:39,488 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:16:39,488 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:16:45,651 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:16:45,651 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:16:45,652 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:16:46,601 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:16:48,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:50,603 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:52,226 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:16:52,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:54,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:16:54,619 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:16:54,619 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:17:05,756 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:17:05,756 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:17:05,757 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:17:06,609 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:17:08,610 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:09,751 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:17:09,752 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:17:10,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:12,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:14,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:22,305 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:17:24,925 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:17:24,925 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:17:25,837 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:17:25,838 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:17:25,838 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:17:26,617 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:17:28,618 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:30,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:32,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:34,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:40,057 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:17:40,057 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:17:45,886 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:17:45,886 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:17:45,887 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:17:46,625 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:17:46,625 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:48,626 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:50,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:52,382 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:17:52,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:54,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:17:55,188 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:17:55,189 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:18:05,983 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:18:05,983 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:18:05,984 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:18:06,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:18:06,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:08,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:10,317 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:18:10,318 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:18:10,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:12,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:14,637 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:22,460 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:18:25,522 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:18:25,522 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:18:26,022 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:18:26,022 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:18:26,023 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:18:26,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:18:26,641 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:28,642 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:30,642 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:32,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:34,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:40,653 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:18:40,653 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:18:46,071 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:18:46,072 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:18:46,072 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:18:46,648 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:18:46,648 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:48,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:50,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:52,538 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:18:52,650 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:54,651 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:18:55,784 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:18:55,784 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:19:06,108 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:19:06,108 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:19:06,109 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:19:06,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:19:08,656 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:10,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:10,913 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:19:10,913 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:19:12,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:14,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:22,615 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:19:26,060 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:19:26,060 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:19:26,190 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:19:26,191 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:19:26,191 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:19:26,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:19:28,664 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:30,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:32,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:34,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:41,216 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:19:41,216 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:19:46,217 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:19:46,217 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:19:46,217 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:19:46,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:19:46,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:48,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:50,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:52,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:52,693 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:19:54,674 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:19:56,350 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:19:56,351 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:20:06,282 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:20:06,283 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:20:06,283 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:20:06,678 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:20:06,678 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:07,678 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:08,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:10,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:11,481 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:20:11,481 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:20:12,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:14,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:22,770 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:20:26,369 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:20:26,369 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:20:26,369 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:20:26,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:20:26,729 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:20:26,730 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:20:27,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:28,687 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:30,688 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:32,688 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:35,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:41,902 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:20:41,903 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:20:46,452 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:20:46,452 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:20:46,453 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:20:46,694 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:20:49,695 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:51,696 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:52,847 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:20:53,696 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:55,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:20:57,032 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:20:57,033 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:21:06,501 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:21:06,501 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:21:06,501 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:21:06,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:21:07,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:09,703 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:11,703 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:12,161 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:21:12,162 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:21:13,704 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:15,705 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:22,926 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:21:26,560 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:21:26,560 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:21:26,561 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:21:26,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:21:27,373 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:21:27,374 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:21:27,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:29,711 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:31,712 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:33,713 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:35,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:42,551 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:21:42,552 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:21:46,639 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:21:46,639 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:21:46,639 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:21:46,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:21:47,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:49,719 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:51,720 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:53,004 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:21:53,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:55,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:21:57,686 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:21:57,686 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:22:06,672 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:22:06,673 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:22:06,673 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:22:06,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:22:07,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:09,727 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:11,728 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:12,843 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:22:12,843 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:22:13,729 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:15,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:23,082 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:22:26,760 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:22:26,760 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:22:26,760 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:22:27,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:22:27,735 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:27,984 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:22:27,984 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:22:29,735 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:31,736 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:33,737 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:35,738 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:43,172 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:22:43,172 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:22:46,795 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:22:46,795 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:22:46,796 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:22:47,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:22:47,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:49,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:51,744 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:53,151 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:22:53,745 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:55,746 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:22:58,329 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:22:58,329 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:23:06,868 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:23:06,869 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:23:06,869 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:23:07,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:23:07,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:09,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:11,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:13,458 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:23:13,458 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:23:13,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:15,753 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:23,221 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:23:26,917 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:23:26,918 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:23:26,918 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:23:27,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:23:27,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:28,601 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:23:28,602 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:23:29,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:31,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:33,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:35,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:43,782 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:23:43,783 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:23:46,970 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:23:46,971 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:23:46,971 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:23:47,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:23:47,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:49,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:51,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:53,294 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:23:53,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:55,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:23:58,914 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:23:58,914 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:24:07,011 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:24:07,011 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:24:07,012 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:24:07,773 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:24:07,773 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:09,774 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:11,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:13,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:14,047 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:24:14,048 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:24:15,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:23,372 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:24:27,087 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:24:27,087 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:24:27,088 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:24:27,781 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:24:27,781 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:29,187 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:24:29,187 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:24:29,782 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:32,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:34,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:36,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:44,321 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:24:44,321 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:24:47,186 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:24:47,186 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:24:47,187 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:24:47,789 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:24:47,789 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:48,790 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:50,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:52,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:53,448 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:24:54,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:56,793 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:24:59,452 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:24:59,453 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:25:07,228 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:25:07,229 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:25:07,229 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:25:07,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:25:07,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:08,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:10,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:12,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:14,584 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:25:14,584 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:25:14,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:16,801 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:23,524 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:25:27,280 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:25:27,280 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:25:27,280 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:25:27,806 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:25:28,806 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:29,778 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:25:29,778 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:25:30,807 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:32,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:34,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:36,810 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:44,909 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:25:44,910 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:25:47,371 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:25:47,372 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:25:47,372 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:25:47,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:25:48,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:50,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:52,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:53,602 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:25:54,818 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:25:56,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:00,044 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:26:00,044 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:26:07,456 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:26:07,456 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:26:07,457 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:26:07,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:26:08,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:10,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:12,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:14,827 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:15,175 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:26:15,175 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:26:16,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:23,682 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:26:27,552 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:26:27,552 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:26:27,553 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:26:27,832 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:26:28,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:30,378 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:26:30,379 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:26:30,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:32,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:34,835 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:36,836 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:45,513 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:26:45,514 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:26:47,591 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:26:47,591 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:26:47,592 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:26:47,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:26:48,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:50,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:52,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:53,760 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:26:54,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:26:56,844 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:00,645 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:27:00,646 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:27:07,626 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:27:07,626 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:27:07,627 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:27:07,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:27:08,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:10,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:12,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:14,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:15,777 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:27:15,778 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:27:16,852 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:23,838 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:27:27,696 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:27:27,696 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:27:27,697 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:27:27,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:27:28,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:30,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:30,916 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:27:30,916 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:27:32,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:34,860 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:36,861 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:46,073 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:27:46,073 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:27:47,746 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:27:47,746 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:27:47,746 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:27:47,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:27:48,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:50,866 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:52,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:53,916 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:27:54,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:27:56,869 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:01,207 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:28:01,207 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:28:07,780 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:28:07,780 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:28:07,781 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:28:07,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:28:08,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:09,874 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:11,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:13,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:15,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:16,340 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:28:16,340 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:28:23,993 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:28:27,835 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:28:27,835 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:28:27,836 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:28:27,881 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:28:28,881 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:29,882 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:31,490 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:28:31,490 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:28:31,883 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:33,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:35,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:46,624 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:28:46,624 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:28:47,972 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:28:47,972 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:28:47,973 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:28:48,889 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:28:48,890 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:49,890 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:51,891 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:53,891 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:54,072 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:28:55,892 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:28:57,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:01,763 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:29:01,763 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:29:08,082 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:29:08,083 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:29:08,083 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:29:08,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:29:08,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:09,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:11,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:13,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:15,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:16,894 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:29:16,894 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:29:17,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:24,149 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:29:28,131 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:29:28,131 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:29:28,131 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:29:28,905 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:29:29,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:31,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:32,029 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:29:32,030 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:29:33,907 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:35,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:37,909 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:47,203 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:29:47,204 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:29:48,200 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:29:48,201 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:29:48,201 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:29:48,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:29:49,914 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:51,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:53,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:54,227 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:29:55,917 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:29:57,918 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:02,341 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:30:02,342 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:30:08,326 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:30:08,327 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:30:08,327 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:30:08,923 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:30:09,923 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:11,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:13,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:15,926 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:17,472 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:30:17,472 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:30:24,306 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:30:28,408 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:30:28,409 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:30:28,409 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:30:28,930 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:30:29,931 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:31,932 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:32,670 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:30:32,671 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:30:33,932 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:35,933 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:47,854 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:30:47,855 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:30:48,457 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:30:48,457 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:30:48,457 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:30:48,938 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:30:49,938 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:51,939 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:53,940 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:30:54,383 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:30:55,941 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:02,985 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:31:02,985 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:31:08,518 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:31:08,518 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:31:08,519 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:31:08,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:31:09,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:11,947 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:13,948 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:15,949 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:17,949 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:18,131 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:31:18,131 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:31:24,461 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:31:28,570 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:31:28,571 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:31:28,571 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:31:28,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:31:29,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:31,955 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:33,267 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:31:33,267 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:31:33,956 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:35,957 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:48,397 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:31:48,398 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:31:48,642 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:31:48,643 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:31:48,643 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:31:48,961 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:31:49,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:51,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:54,539 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:31:54,964 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:31:56,965 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:03,528 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:32:03,528 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:32:08,689 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:32:08,689 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:32:08,690 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:32:08,969 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:32:09,970 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:10,970 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:12,971 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:14,972 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:16,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:18,660 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:32:18,661 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:32:24,615 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:32:28,735 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:32:28,735 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:32:28,736 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:32:28,977 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:32:29,978 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:30,978 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:32,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:33,790 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:32:33,790 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:32:34,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:36,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:48,770 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:32:48,771 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:32:48,771 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:32:48,932 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:32:48,932 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:32:48,985 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:32:49,985 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:50,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:52,987 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:54,693 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:32:54,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:32:56,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:04,077 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:33:04,078 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:33:08,851 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:33:08,852 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:33:08,852 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:33:08,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:33:09,993 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:10,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:12,995 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:14,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:16,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:19,241 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:33:19,242 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:33:24,771 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:33:28,900 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:33:28,900 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:33:28,901 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:33:29,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:33:30,001 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:31,002 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:33,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:34,386 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:33:34,386 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:33:35,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:37,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:48,963 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:33:48,964 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:33:48,964 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:33:49,008 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:33:49,523 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:33:49,524 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:33:50,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:51,009 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:53,010 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:54,848 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:33:55,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:33:57,012 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:04,682 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:34:04,682 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:34:09,054 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:34:09,054 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:34:09,055 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:34:10,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:34:10,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:11,017 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:13,018 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:15,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:17,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:19,812 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:34:19,812 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:34:24,927 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:34:29,092 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:34:29,092 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:34:29,093 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:34:30,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:34:30,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:31,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:33,025 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:34,942 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:34:34,942 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:34:35,026 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:37,027 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:49,200 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:34:49,200 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:34:49,201 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:34:50,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:34:50,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:50,130 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:34:50,130 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:34:51,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:53,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:55,004 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:34:55,033 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:34:57,034 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:05,263 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:35:05,264 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:35:09,319 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:35:09,319 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:35:09,319 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:35:10,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:35:11,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:13,040 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:15,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:17,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:20,396 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:35:20,396 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:35:25,082 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:35:29,378 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:35:29,379 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:35:29,379 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:35:30,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:35:31,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:34,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:35,525 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:35:35,526 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:35:36,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:38,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:49,434 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:35:49,435 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:35:49,435 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:35:50,053 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:35:50,673 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:35:50,674 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:35:51,053 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:52,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:54,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:55,159 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:35:56,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:35:58,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:05,807 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:36:05,808 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:36:09,482 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:36:09,482 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:36:09,482 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:36:10,061 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:36:11,062 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:12,062 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:14,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:16,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:18,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:20,937 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:36:20,938 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:36:25,238 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:36:29,551 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:36:29,552 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:36:29,552 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:36:30,069 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:36:31,069 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:32,070 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:34,071 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:36,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:36,098 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:36:36,099 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:36:38,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:49,594 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:36:49,594 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:36:49,594 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:36:50,077 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:36:51,077 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:51,236 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:36:51,237 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:36:52,078 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:54,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:55,315 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:36:56,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:36:58,080 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:06,414 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:37:06,414 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:37:09,646 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:37:09,646 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:37:09,647 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:37:10,085 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:37:11,085 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:12,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:14,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:16,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:18,088 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:21,545 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:37:21,545 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:37:25,395 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:37:29,713 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:37:29,713 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:37:29,714 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:37:30,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:37:31,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:32,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:34,094 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:36,095 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:36,684 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:37:36,684 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:37:38,096 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:49,784 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:37:49,784 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:37:49,785 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:37:50,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:37:51,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:51,847 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:37:51,848 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:37:52,102 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:54,102 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:55,477 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:37:56,103 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:37:58,104 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:07,034 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:38:07,034 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:38:09,847 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:38:09,847 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:38:09,848 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:38:10,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:38:11,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:12,110 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:14,111 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:16,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:18,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:22,176 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:38:22,176 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:38:25,554 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:38:29,989 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:38:29,990 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:38:29,990 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:38:30,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:38:31,117 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:32,118 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:34,118 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:36,119 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:37,311 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:38:37,312 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:38:38,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:50,040 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:38:50,041 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:38:50,041 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:38:50,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:38:51,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:52,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:52,491 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:38:52,491 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:38:54,126 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:55,632 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:38:56,127 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:38:58,128 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:07,652 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:39:07,652 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:39:10,089 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:39:10,090 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:39:10,090 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:39:10,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:39:11,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:13,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:15,134 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:17,135 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:19,136 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:22,791 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:39:22,791 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:39:25,709 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:39:30,145 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:39:30,145 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:39:30,146 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:39:31,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:39:32,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:33,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:35,142 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:37,142 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:38,011 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:39:38,011 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:39:39,143 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:50,214 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:39:50,215 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:39:50,215 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:39:51,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:39:51,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:52,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:53,149 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:53,155 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:39:53,155 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:39:55,149 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:55,788 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:39:57,150 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:39:59,151 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:08,313 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:40:08,314 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:40:10,278 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:40:10,279 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:40:10,281 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:40:11,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:40:12,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:13,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:15,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:17,158 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:19,159 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:23,448 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:40:23,448 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:40:25,865 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:40:30,346 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:40:30,346 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:40:30,347 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:40:31,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:40:32,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:33,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:35,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:37,166 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:38,581 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:40:38,581 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:40:39,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:50,398 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:40:50,398 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:40:50,398 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:40:51,171 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:40:51,171 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:52,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:53,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:53,711 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:40:53,712 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:40:55,173 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:55,943 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:40:57,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:40:59,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:08,904 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:41:08,904 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:41:10,486 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:41:10,487 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:41:10,487 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:41:11,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:41:11,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:12,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:13,180 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:15,181 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:17,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:19,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:24,034 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:41:24,034 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:41:26,021 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:41:30,615 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:41:30,616 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:41:30,616 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:41:31,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:41:31,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:32,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:33,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:35,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:37,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:39,163 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:41:39,163 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:41:39,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:50,650 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:41:50,650 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:41:50,651 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:41:51,195 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:41:52,195 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:53,196 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:54,305 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:41:54,306 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:41:55,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:56,098 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:41:57,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:41:59,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:09,457 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:42:09,457 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:42:10,743 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:42:10,743 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:42:10,744 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:42:11,202 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:42:12,203 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:13,203 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:15,204 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:17,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:19,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:24,588 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:42:24,588 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:42:26,175 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:42:30,788 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:42:30,789 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:42:30,789 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:42:31,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:42:32,210 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:33,211 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:35,212 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:37,212 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:39,213 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:39,738 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:42:39,738 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:42:50,832 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:42:50,832 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:42:50,832 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:42:51,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:42:51,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:52,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:53,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:54,872 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:42:54,873 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:42:55,219 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:42:56,252 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:42:57,220 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:00,222 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:10,002 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:43:10,002 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:43:10,909 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:43:10,910 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:43:10,910 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:43:11,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:43:12,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:14,227 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:16,228 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:18,228 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:20,229 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:25,133 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:43:25,134 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:43:26,325 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:43:31,012 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:43:31,012 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:43:31,013 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:43:31,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:43:32,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:34,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:36,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:38,236 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:40,237 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:40,262 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:43:40,262 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:43:51,060 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:43:51,060 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:43:51,061 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:43:51,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:43:52,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:54,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:55,391 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:43:55,392 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:43:56,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:43:56,401 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:43:58,244 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:00,245 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:10,530 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:44:10,530 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:44:11,113 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:44:11,113 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:44:11,114 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:44:11,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:44:12,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:14,251 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:16,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:18,253 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:20,253 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:25,679 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:44:25,680 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:44:26,472 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:44:31,164 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:44:31,164 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:44:31,165 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:44:31,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:44:32,258 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:34,259 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:36,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:38,261 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:40,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:40,812 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:44:40,812 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:44:51,278 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:44:51,279 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:44:51,279 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:44:52,267 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:44:52,267 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:54,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:55,945 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:44:55,945 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:44:56,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:44:56,541 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:44:58,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:00,271 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:11,080 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:45:11,080 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:45:11,363 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:45:11,363 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:45:11,364 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:45:12,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:45:12,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:14,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:16,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:18,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:20,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:26,211 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:45:26,211 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:45:26,614 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:45:31,423 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:45:31,423 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:45:31,423 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:45:32,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:45:32,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:33,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:34,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:36,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:38,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:40,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:41,344 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:45:41,344 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:45:51,543 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:45:51,543 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:45:51,544 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:45:52,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:45:52,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:53,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:54,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:56,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:45:56,473 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:45:56,477 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:45:56,689 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:45:58,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:00,295 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:11,582 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:46:11,582 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:46:11,582 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:46:11,621 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:46:11,621 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:46:12,300 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:46:12,300 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:13,300 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:14,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:16,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:18,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:20,303 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:26,754 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:46:26,754 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:46:26,891 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:46:31,619 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:46:31,619 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:46:31,619 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:46:32,307 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:46:32,307 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:33,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:34,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:36,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:39,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:41,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:41,893 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:46:41,893 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:46:51,786 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:46:51,786 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:46:51,787 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:46:52,315 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:46:53,316 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:55,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:56,838 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:46:57,021 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:46:57,021 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:46:57,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:46:59,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:01,319 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:11,822 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:47:11,822 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:47:11,823 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:47:12,226 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:47:12,227 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:47:12,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:47:13,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:15,325 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:17,326 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:19,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:21,328 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:26,913 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:47:27,411 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:47:27,411 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:47:31,891 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:47:31,892 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:47:31,894 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:47:32,332 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:47:33,333 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:35,334 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:37,334 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:39,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:41,336 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:42,543 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:47:42,543 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:47:51,930 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:47:51,931 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:47:51,931 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:47:52,340 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:47:53,340 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:55,341 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:56,990 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:47:57,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:47:57,683 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:47:57,684 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:47:59,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:01,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:12,007 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:48:12,007 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:48:12,010 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:48:12,349 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:48:12,826 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:48:12,826 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:48:13,349 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:15,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:17,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:19,352 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:21,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:27,067 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:48:27,997 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:48:27,997 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:48:32,147 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:48:32,147 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:48:32,148 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:48:32,357 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:48:33,357 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:35,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:37,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:39,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:41,361 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:43,128 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:48:43,128 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:48:52,217 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:48:52,218 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:48:52,220 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:48:52,365 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:48:53,366 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:55,367 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:57,145 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:48:57,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:48:58,259 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:48:58,259 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:48:59,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:01,370 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:12,285 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:49:12,285 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:49:12,285 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:49:12,375 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:49:13,375 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:13,547 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:49:13,547 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:49:15,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:17,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:19,378 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:21,379 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:27,223 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:49:28,723 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:49:28,723 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:49:32,366 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:49:32,366 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:49:32,366 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:49:32,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:49:33,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:35,384 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:37,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:39,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:41,387 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:43,856 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:49:43,857 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:49:52,460 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:49:52,460 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:49:52,460 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:49:53,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:49:53,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:55,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:57,301 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:49:57,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:49:58,997 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:49:58,998 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:49:59,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:01,394 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:12,525 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:50:12,526 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:50:12,526 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:50:13,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:50:13,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:14,130 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:50:14,131 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:50:14,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:16,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:18,400 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:20,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:22,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:27,379 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:50:29,289 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:50:29,289 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:50:32,612 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:50:32,612 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:50:32,613 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:50:33,406 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:50:34,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:36,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:38,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:40,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:42,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:44,421 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:50:44,422 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:50:52,736 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:50:52,736 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:50:52,736 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:50:53,414 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:50:54,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:56,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:57,456 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:50:58,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:50:59,552 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:50:59,552 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:51:00,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:02,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:12,817 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:51:12,818 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:51:12,818 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:51:13,422 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:51:14,422 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:14,729 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:51:14,730 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:51:16,423 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:18,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:20,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:22,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:27,534 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:51:29,936 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:51:29,936 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:51:32,892 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:51:32,892 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:51:32,892 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:51:33,430 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:51:34,431 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:36,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:38,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:40,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:42,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:45,066 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:51:45,067 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:51:52,942 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:51:52,942 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:51:52,942 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:51:53,438 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:51:54,438 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:56,439 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:51:57,611 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:51:58,440 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:00,198 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:52:00,198 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:52:00,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:02,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:12,976 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:52:12,976 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:52:12,977 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:52:13,446 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:52:14,446 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:15,338 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:52:15,339 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:52:16,447 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:18,448 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:20,448 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:27,690 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:52:30,517 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:52:30,517 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:52:33,069 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:52:33,069 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:52:33,070 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:52:33,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:52:34,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:36,454 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:38,455 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:40,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:45,648 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:52:45,649 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:52:53,177 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:52:53,177 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:52:53,178 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:52:53,461 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:52:54,461 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:56,462 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:52:57,767 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:52:58,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:00,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:00,782 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:53:00,782 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:53:02,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:13,224 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:53:13,224 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:53:13,225 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:53:13,468 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:53:14,469 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:15,927 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:53:15,927 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:53:16,469 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:18,470 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:20,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:22,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:27,845 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:53:31,082 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:53:31,083 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:53:33,299 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:53:33,299 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:53:33,300 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:53:33,476 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:53:34,476 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:36,477 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:38,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:40,479 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:42,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:46,236 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:53:46,237 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:53:53,332 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:53:53,332 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:53:53,333 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:53:53,484 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:53:54,485 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:55,485 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:57,486 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:53:57,923 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:53:59,487 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:01,369 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:54:01,369 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:54:01,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:13,380 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:54:13,380 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:54:13,381 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:54:13,492 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:54:14,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:15,493 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:16,505 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:54:16,505 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:54:17,494 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:19,495 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:21,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:28,001 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:54:31,644 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:54:31,644 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:54:33,426 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:54:33,427 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:54:33,427 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:54:33,501 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:54:35,502 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:37,503 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:39,504 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:41,505 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:46,798 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:54:46,798 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:54:53,500 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:54:53,500 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:54:53,501 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:54:53,509 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:54:55,510 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:57,511 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:54:58,079 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:54:59,512 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:01,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:01,930 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:55:01,931 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:55:13,558 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:55:13,558 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:55:13,559 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:55:14,518 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:55:15,518 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:17,095 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:55:17,095 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:55:17,519 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:19,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:21,521 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:28,156 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:55:32,301 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:55:32,302 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:55:33,614 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:55:33,615 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:55:33,615 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:55:34,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:55:35,526 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:37,527 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:39,528 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:41,529 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:47,434 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:55:47,434 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:55:53,751 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:55:53,751 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:55:53,752 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:55:54,534 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:55:55,534 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:57,535 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:55:58,235 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:55:59,536 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:01,536 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:02,567 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:56:02,567 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:56:13,794 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:56:13,795 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:56:13,795 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:56:14,541 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:56:15,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:17,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:17,712 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:56:17,712 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:56:19,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:21,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:28,313 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:56:32,865 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:56:32,865 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:56:33,830 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:56:33,831 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:56:33,831 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:56:34,549 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:56:35,549 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:37,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:39,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:41,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:48,000 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:56:48,001 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:56:53,883 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:56:53,883 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:56:53,884 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:56:54,557 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:56:55,557 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:57,558 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:56:58,390 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:56:59,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:02,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:03,135 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:57:03,136 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:57:13,928 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:57:13,928 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:57:13,928 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:57:14,564 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:57:15,565 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:16,565 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:18,267 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:57:18,268 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:57:18,566 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:20,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:22,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:28,466 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:57:33,397 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:57:33,398 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:57:33,992 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:57:33,993 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:57:33,993 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:57:34,572 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:57:35,573 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:36,573 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:38,574 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:40,575 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:42,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:48,530 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:57:48,531 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:57:54,051 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:57:54,051 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:57:54,052 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:57:54,580 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:57:55,581 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:56,581 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:57:58,545 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:57:58,582 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:00,583 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:02,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:03,661 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:58:03,662 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:58:14,103 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:58:14,103 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:58:14,103 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:58:14,588 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:58:15,589 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:16,590 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:18,590 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:18,796 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:58:18,797 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:58:20,591 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:22,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:28,622 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:58:33,926 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:58:33,926 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:58:34,140 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:58:34,140 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:58:34,141 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:58:34,597 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:58:35,597 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:36,598 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:38,598 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:40,599 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:42,600 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:49,055 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:58:49,055 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:58:54,267 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:58:54,268 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:58:54,268 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:58:54,605 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:58:55,605 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:56,606 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:58,606 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:58:58,695 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:59:00,607 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:02,608 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:04,189 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:59:04,190 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:59:14,319 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:59:14,320 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:59:14,320 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:59:14,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:59:15,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:16,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:18,614 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:19,319 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:59:19,320 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:59:20,615 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:22,615 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:28,764 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:59:34,358 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:59:34,358 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:59:34,359 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:59:34,452 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:59:34,452 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:59:34,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:59:35,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:36,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:38,622 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:40,622 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:42,623 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:49,600 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 11:59:49,601 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 11:59:54,447 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 11:59:54,448 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 11:59:54,448 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 11:59:54,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 11:59:55,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:57,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 11:59:58,834 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 11:59:59,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:01,630 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:03,631 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:04,730 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:00:04,730 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:00:14,641 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:00:14,641 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:00:14,641 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:00:15,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:00:15,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:16,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:17,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:19,637 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:19,860 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:00:19,860 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:00:21,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:23,639 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:28,904 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:00:34,678 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:00:34,679 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:00:34,679 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:00:35,051 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:00:35,051 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:00:35,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:00:36,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:37,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:39,645 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:41,645 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:43,646 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:50,238 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:00:50,239 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:00:54,729 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:00:54,729 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:00:54,730 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:00:55,651 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:00:56,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:57,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:00:58,974 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:00:59,653 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:01,654 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:03,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:05,373 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:01:05,374 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:01:14,778 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:01:14,778 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:01:14,778 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:01:15,660 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:01:16,660 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:17,661 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:19,662 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:20,519 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:01:20,520 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:01:21,662 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:23,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:29,052 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:01:34,835 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:01:34,835 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:01:34,836 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:01:35,668 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:01:35,689 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:01:35,690 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:01:36,668 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:37,669 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:39,670 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:41,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:43,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:50,830 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:01:50,831 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:01:54,871 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:01:54,872 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:01:54,872 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:01:55,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:01:55,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:56,677 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:57,678 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:01:59,128 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:01:59,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:01,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:03,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:05,959 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:02:05,959 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:02:14,921 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:02:14,921 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:02:14,922 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:02:15,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:02:16,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:17,686 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:19,687 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:21,087 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:02:21,088 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:02:21,688 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:23,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:29,204 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:02:34,956 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:02:34,956 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:02:34,957 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:02:35,693 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:02:36,271 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:02:36,271 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:02:36,694 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:37,694 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:39,695 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:41,696 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:43,697 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:51,461 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:02:51,461 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:02:55,088 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:02:55,088 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:02:55,089 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:02:55,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:02:55,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:56,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:57,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:02:59,282 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:02:59,703 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:01,704 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:03,705 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:06,591 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:03:06,592 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:03:15,173 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:03:15,173 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:03:15,174 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:03:15,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:03:15,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:16,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:17,710 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:19,711 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:21,712 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:21,730 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:03:21,731 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:03:23,713 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:29,359 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:03:35,241 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:03:35,241 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:03:35,241 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:03:35,717 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:03:36,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:36,880 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:03:36,880 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:03:37,718 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:39,719 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:41,720 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:43,720 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:52,024 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:03:52,025 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:03:55,294 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:03:55,294 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:03:55,295 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:03:55,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:03:55,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:56,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:57,726 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:03:59,437 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:03:59,727 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:01,728 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:03,729 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:07,158 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:04:07,159 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:04:15,331 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:04:15,332 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:04:15,332 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:04:15,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:04:15,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:16,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:17,734 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:19,735 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:21,736 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:22,293 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:04:22,294 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:04:23,737 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:29,517 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:04:35,388 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:04:35,389 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:04:35,389 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:04:35,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:04:35,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:36,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:37,461 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:04:37,461 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:04:37,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:39,743 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:42,744 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:44,745 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:52,612 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:04:52,612 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:04:55,504 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:04:55,504 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:04:55,505 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:04:55,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:04:56,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:58,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:04:59,597 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:05:00,752 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:02,753 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:04,754 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:07,742 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:05:07,742 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:05:15,549 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:05:15,550 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:05:15,550 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:05:15,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:05:16,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:18,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:20,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:22,761 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:22,873 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:05:22,873 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:05:24,762 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:29,676 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:05:35,659 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:05:35,659 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:05:35,660 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:05:35,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:05:36,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:38,011 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:05:38,011 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:05:38,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:40,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:42,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:44,770 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:53,177 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:05:53,177 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:05:55,713 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:05:55,713 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:05:55,714 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:05:55,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:05:56,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:58,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:05:59,753 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:06:00,778 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:02,779 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:04,780 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:08,305 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:06:08,305 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:06:15,758 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:06:15,758 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:06:15,759 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:06:15,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:06:16,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:18,786 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:20,787 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:22,788 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:23,448 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:06:23,448 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:06:24,788 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:29,831 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:06:35,830 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:06:35,831 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:06:35,831 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:06:36,793 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:06:37,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:38,588 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:06:38,588 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:06:38,794 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:40,795 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:42,796 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:44,797 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:53,730 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:06:53,731 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:06:55,908 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:06:55,908 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:06:55,909 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:06:56,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:06:57,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:58,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:06:59,908 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:07:00,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:02,805 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:04,806 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:08,862 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:07:08,862 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:07:15,984 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:07:15,985 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:07:15,985 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:07:16,811 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:07:16,811 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:17,812 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:18,812 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:20,813 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:22,814 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:24,000 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:07:24,000 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:07:24,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:29,988 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:07:36,103 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:07:36,104 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:07:36,104 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:07:36,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:07:36,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:37,820 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:38,820 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:39,186 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:07:39,186 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:07:40,821 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:42,822 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:44,823 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:54,338 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:07:54,339 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:07:56,191 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:07:56,191 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:07:56,192 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:07:56,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:07:56,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:57,829 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:07:58,829 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:00,065 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:08:00,830 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:02,831 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:04,832 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:09,470 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:08:09,471 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:08:16,260 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:08:16,260 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:08:16,261 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:08:16,837 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:08:16,837 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:17,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:18,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:20,839 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:22,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:24,598 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:08:24,598 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:08:24,841 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:30,142 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:08:36,304 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:08:36,304 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:08:36,305 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:08:36,845 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:08:36,846 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:37,846 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:38,846 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:39,774 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:08:39,775 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:08:41,847 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:43,848 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:45,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:54,910 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:08:54,911 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:08:56,383 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:08:56,383 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:08:56,384 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:08:56,854 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:08:57,855 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:08:59,856 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:00,219 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:09:01,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:03,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:05,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:10,040 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:09:10,040 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:09:16,500 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:09:16,501 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:09:16,501 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:09:16,864 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:09:17,864 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:19,865 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:21,866 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:23,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:25,172 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:09:25,172 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:09:25,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:30,297 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:09:36,577 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:09:36,578 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:09:36,578 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:09:36,872 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:09:37,873 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:39,874 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:40,377 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:09:40,378 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:09:41,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:43,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:45,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:55,546 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:09:55,547 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:09:56,624 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:09:56,624 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:09:56,625 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:09:56,881 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:09:57,881 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:09:59,882 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:00,375 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:10:01,883 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:03,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:05,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:10,677 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:10:10,677 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:10:16,663 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:10:16,663 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:10:16,664 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:10:16,889 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:10:17,890 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:19,891 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:21,892 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:23,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:25,812 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:10:25,813 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:10:25,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:30,453 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:10:36,707 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:10:36,707 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:10:36,708 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:10:36,897 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:10:37,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:38,898 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:39,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:40,942 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:10:40,943 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:10:41,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:43,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:45,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:56,070 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:10:56,071 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:10:56,803 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:10:56,803 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:10:56,804 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:10:56,906 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:10:57,907 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:58,907 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:10:59,907 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:00,531 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:11:01,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:03,909 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:05,910 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:11,210 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:11:11,211 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:11:16,868 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:11:16,868 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:11:16,869 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:11:16,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:11:17,915 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:18,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:19,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:21,917 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:23,918 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:25,919 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:26,350 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:11:26,350 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:11:30,609 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:11:36,940 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:11:36,941 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:11:36,941 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:11:37,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:11:37,924 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:38,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:39,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:41,488 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:11:41,488 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:11:41,926 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:43,927 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:45,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:56,618 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:11:56,619 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:11:57,009 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:11:57,009 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:11:57,010 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:11:57,933 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:11:57,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:58,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:11:59,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:00,686 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:12:01,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:03,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:05,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:11,762 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:12:11,762 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:12:17,058 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:12:17,058 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:12:17,059 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:12:17,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:12:17,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:18,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:19,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:21,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:23,945 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:25,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:26,893 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:12:26,893 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:12:30,765 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:12:37,137 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:12:37,137 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:12:37,138 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:12:37,951 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:12:37,951 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:38,951 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:39,952 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:41,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:42,073 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:12:42,074 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:12:43,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:45,954 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:12:57,194 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:12:57,195 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:12:57,195 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:12:57,250 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:12:57,250 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:12:57,959 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:12:58,959 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:00,842 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:13:00,960 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:02,961 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:04,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:06,963 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:12,421 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:13:12,421 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:13:17,249 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:13:17,249 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:13:17,250 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:13:17,968 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:13:18,969 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:20,970 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:22,971 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:24,972 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:26,973 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:27,554 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:13:27,554 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:13:30,921 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:13:37,310 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:13:37,311 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:13:37,311 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:13:37,978 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:13:38,978 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:40,979 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:42,685 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:13:42,685 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:13:42,980 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:44,981 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:46,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:13:57,381 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:13:57,381 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:13:57,382 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:13:57,831 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:13:57,831 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:13:57,986 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:13:58,987 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:00,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:00,999 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:14:02,988 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:04,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:06,990 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:12,966 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:14:12,967 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:14:17,425 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:14:17,425 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:14:17,426 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:14:17,994 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:14:18,995 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:20,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:22,996 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:24,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:28,096 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:14:28,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:14:31,073 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:14:37,486 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:14:37,487 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:14:37,487 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:14:38,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:14:39,003 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:41,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:43,005 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:43,225 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:14:43,226 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:14:45,006 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:14:57,522 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:14:57,522 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:14:57,523 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:14:58,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:14:58,388 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:14:58,389 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:14:59,011 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:01,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:01,143 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:15:03,014 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:05,015 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:13,526 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:15:13,526 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:15:17,570 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:15:17,571 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:15:17,571 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:15:18,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:15:19,021 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:20,021 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:21,022 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:23,023 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:25,024 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:28,657 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:15:28,657 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:15:31,212 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:15:37,618 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:15:37,618 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:15:37,619 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:15:38,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:15:39,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:40,030 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:41,030 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:43,031 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:43,793 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:15:43,794 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:15:45,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:15:57,661 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:15:57,662 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:15:57,662 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:15:58,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:15:58,926 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:15:58,926 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:15:59,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:00,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:01,284 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:16:02,040 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:04,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:06,042 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:14,085 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:16:14,086 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:16:17,698 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:16:17,699 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:16:17,699 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:16:18,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:16:20,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:22,050 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:24,051 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:26,052 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:29,216 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:16:29,216 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:16:31,359 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:16:37,762 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:16:37,762 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:16:37,763 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:16:38,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:16:40,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:42,059 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:44,060 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:44,350 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:16:44,351 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:16:46,061 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:16:57,799 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:16:57,799 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:16:57,800 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:16:58,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:16:59,537 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:16:59,537 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:17:00,067 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:01,436 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:17:02,068 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:04,069 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:06,070 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:14,673 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:17:14,673 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:17:17,858 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:17:17,859 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:17:17,859 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:17:18,075 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:17:20,076 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:22,077 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:24,078 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:26,079 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:29,816 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:17:29,816 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:17:31,526 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:17:37,895 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:17:37,895 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:17:37,896 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:17:38,084 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:17:40,085 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:42,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:44,086 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:44,947 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:17:44,947 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:17:46,087 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:17:57,953 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:17:57,954 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:17:57,954 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:17:58,092 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:18:00,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:00,108 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:18:00,108 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:18:01,614 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:18:02,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:04,094 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:06,095 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:15,259 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:18:15,259 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:18:18,036 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:18:18,037 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:18:18,037 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:18:18,100 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:18:20,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:22,102 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:24,103 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:26,104 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:30,389 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:18:30,389 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:18:31,692 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:18:38,113 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:18:38,113 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:18:38,114 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:18:39,110 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:18:40,110 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:42,111 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:44,112 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:45,525 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:18:45,526 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:18:46,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:18:58,172 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:18:58,172 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:18:58,173 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:18:59,119 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:19:00,119 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:00,658 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:19:00,659 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:19:01,772 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:19:02,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:04,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:06,150 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:19:06,150 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:19:06,151 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:19:07,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:19:08,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:10,123 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:12,124 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:14,125 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:15,789 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:19:15,789 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:19:26,206 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:19:26,207 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:19:26,207 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:19:27,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:19:28,130 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:30,131 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:30,918 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:19:30,918 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:19:31,851 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:19:32,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:34,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:46,048 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:19:46,049 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:19:46,264 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:19:46,265 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:19:46,265 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:19:47,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:19:48,138 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:50,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:52,139 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:19:54,140 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:01,180 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:20:01,181 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:20:01,929 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:20:06,380 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:20:06,381 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:20:06,381 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:20:07,146 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:20:08,146 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:10,147 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:12,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:14,148 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:16,311 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:20:16,312 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:20:26,420 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:20:26,421 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:20:26,421 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:20:27,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:20:28,154 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:30,155 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:31,444 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:20:31,444 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:20:32,007 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:20:32,156 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:34,157 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:46,468 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:20:46,469 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:20:46,469 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:20:46,579 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:20:46,579 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:20:47,162 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:20:48,163 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:50,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:52,164 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:20:54,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:01,729 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:21:01,729 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:21:02,084 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:21:06,564 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:21:06,565 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:21:06,565 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:21:07,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:21:08,170 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:10,171 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:12,172 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:14,173 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:16,886 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:21:16,886 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:21:26,614 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:21:26,614 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:21:26,615 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:21:27,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:21:28,179 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:31,181 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:32,021 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:21:32,022 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:21:32,162 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:21:33,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:35,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:46,662 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:21:46,663 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:21:46,663 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:21:47,174 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:21:47,174 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:21:47,187 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:21:49,188 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:51,189 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:53,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:21:55,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:02,239 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:22:02,315 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:22:02,315 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:22:06,779 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:22:06,779 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:22:06,780 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:22:07,196 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:22:09,197 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:11,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:13,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:15,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:17,448 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:22:17,448 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:22:26,850 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:22:26,851 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:22:26,851 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:22:27,205 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:22:29,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:31,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:32,318 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:22:32,579 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:22:32,579 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:22:33,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:35,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:46,901 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:22:46,902 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:22:46,902 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:22:47,214 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:22:47,741 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:22:47,741 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:22:49,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:51,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:53,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:22:55,218 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:02,396 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:23:02,876 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:23:02,877 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:23:06,952 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:23:06,953 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:23:06,953 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:23:07,223 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:23:09,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:11,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:13,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:15,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:18,009 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:23:18,009 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:23:26,989 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:23:26,990 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:23:26,990 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:23:27,231 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:23:29,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:31,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:32,474 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:23:33,196 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:23:33,197 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:23:33,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:35,235 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:47,074 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:23:47,075 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:23:47,075 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:23:47,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:23:48,335 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:23:48,335 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:23:49,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:51,243 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:53,244 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:23:55,245 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:02,551 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:24:03,476 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:24:03,476 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:24:07,120 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:24:07,121 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:24:07,121 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:24:07,250 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:24:09,251 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:11,252 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:13,253 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:15,254 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:18,609 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:24:18,610 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:24:27,196 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:24:27,197 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:24:27,197 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:24:27,259 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:24:29,260 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:31,261 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:32,630 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:24:33,261 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:33,747 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:24:33,747 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:24:35,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:47,256 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:24:47,256 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:24:47,257 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:24:47,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:24:48,880 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:24:48,880 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:24:49,268 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:51,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:53,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:24:55,271 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:02,707 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:25:04,018 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:25:04,018 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:25:07,375 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:25:07,375 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:25:07,376 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:25:08,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:25:09,276 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:11,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:13,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:15,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:19,155 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:25:19,155 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:25:27,445 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:25:27,445 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:25:27,446 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:25:28,283 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:25:29,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:31,284 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:32,785 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:25:33,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:34,291 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:25:34,291 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:25:35,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:47,520 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:25:47,520 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:25:47,520 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:25:48,291 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:25:49,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:49,493 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:25:49,493 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:25:51,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:53,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:25:55,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:02,861 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:26:04,632 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:26:04,633 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:26:07,554 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:26:07,554 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:26:07,555 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:26:08,299 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:26:09,299 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:10,300 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:12,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:14,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:16,303 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:19,763 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:26:19,763 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:26:27,610 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:26:27,610 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:26:27,611 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:26:28,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:26:29,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:30,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:32,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:32,930 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:26:34,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:34,894 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:26:34,895 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:26:36,311 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:47,659 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:26:47,660 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:26:47,660 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:26:48,315 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:26:49,315 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:50,060 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:26:50,061 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:26:50,316 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:52,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:54,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:26:56,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:03,005 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:27:05,471 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:27:05,472 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:27:07,694 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:27:07,694 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:27:07,695 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:27:08,323 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:27:09,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:10,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:12,325 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:14,326 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:16,327 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:20,601 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:27:20,601 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:27:27,729 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:27:27,729 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:27:27,730 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:27:28,331 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:27:30,332 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:32,333 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:33,080 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:27:34,334 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:35,730 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:27:35,731 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:27:36,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:47,788 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:27:47,788 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:27:47,789 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:27:48,340 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:27:50,341 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:50,929 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:27:50,929 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:27:52,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:54,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:27:56,344 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:03,158 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:28:06,094 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:28:06,095 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:28:07,846 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:28:07,846 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:28:07,847 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:28:08,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:28:10,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:12,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:14,352 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:16,353 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:21,228 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:28:21,228 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:28:27,883 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:28:27,884 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:28:27,884 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:28:28,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:28:30,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:32,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:33,233 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:28:34,360 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:36,359 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:28:36,360 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:28:36,362 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:47,944 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:28:47,944 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:28:47,945 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:28:48,367 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:28:50,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:51,496 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:28:51,496 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:28:52,369 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:54,370 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:28:56,370 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:03,311 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:29:06,628 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:29:06,628 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:29:08,028 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:29:08,029 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:29:08,029 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:29:08,375 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:29:10,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:12,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:14,377 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:16,378 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:21,766 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:29:21,767 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:29:28,073 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:29:28,073 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:29:28,074 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:29:28,382 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:29:30,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:32,384 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:33,397 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:29:34,385 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:36,386 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:36,900 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:29:36,901 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:29:48,119 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:29:48,120 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:29:48,120 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:29:48,390 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:29:50,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:52,033 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:29:52,033 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:29:52,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:54,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:29:56,393 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:03,494 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:30:07,165 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:30:07,165 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:30:08,233 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:30:08,234 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:30:08,234 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:30:08,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:30:10,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:12,400 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:14,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:16,402 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:22,294 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:30:22,295 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:30:28,277 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:30:28,278 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:30:28,278 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:30:28,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:30:30,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:32,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:33,576 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:30:34,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:36,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:37,427 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:30:37,428 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:30:48,356 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:30:48,356 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:30:48,357 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:30:48,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:30:50,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:52,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:52,560 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:30:52,560 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:30:54,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:30:56,419 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:03,654 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:31:07,690 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:31:07,690 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:31:08,402 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:31:08,402 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:31:08,402 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:31:08,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:31:10,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:11,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:13,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:15,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:17,428 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:22,820 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:31:22,820 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:31:28,447 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:31:28,448 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:31:28,448 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:31:29,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:31:30,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:31,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:33,435 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:33,732 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:31:35,436 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:37,437 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:37,953 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:31:37,954 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:31:48,558 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:31:48,558 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:31:48,558 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:31:49,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:31:50,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:51,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:53,139 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:31:53,139 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:31:53,444 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:55,445 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:31:57,446 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:03,810 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:32:08,270 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:32:08,270 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:32:08,592 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:32:08,593 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:32:08,593 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:32:09,452 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:32:10,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:11,453 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:13,454 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:15,455 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:17,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:23,399 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:32:23,399 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:32:28,638 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:32:28,638 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:32:28,638 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:32:29,461 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:32:31,462 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:33,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:33,887 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:32:35,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:37,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:38,535 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:32:38,535 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:32:48,702 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:32:48,703 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:32:48,703 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:32:49,470 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:32:49,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:51,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:53,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:53,766 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:32:53,766 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:32:55,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:32:57,474 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:03,965 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:33:08,785 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:33:08,785 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:33:08,786 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:33:08,911 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:33:08,912 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:33:09,479 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:33:09,479 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:11,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:13,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:15,482 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:17,482 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:24,086 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:33:24,086 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:33:28,878 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:33:28,878 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:33:28,879 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:33:29,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:33:31,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:33,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:34,043 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:33:35,490 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:37,491 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:39,217 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:33:39,218 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:33:48,932 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:33:48,932 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:33:48,932 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:33:49,496 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:33:51,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:53,498 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:54,350 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:33:54,350 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:33:55,499 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:33:57,500 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:04,121 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:34:09,014 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:34:09,015 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:34:09,015 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:34:09,504 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:34:09,510 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:34:09,510 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:34:11,505 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:13,506 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:15,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:17,508 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:24,687 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:34:24,687 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:34:29,055 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:34:29,055 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:34:29,056 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:34:29,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:34:29,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:31,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:33,515 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:34,199 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:34:35,516 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:37,517 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:39,818 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:34:39,819 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:34:49,293 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:34:49,293 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:34:49,294 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:34:49,521 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:34:49,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:51,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:53,523 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:54,952 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:34:54,952 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:34:55,524 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:34:57,525 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:04,277 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:35:09,396 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:35:09,396 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:35:09,396 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:35:09,530 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:35:10,272 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:35:10,273 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:35:10,530 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:11,531 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:12,531 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:14,532 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:16,533 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:18,534 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:25,404 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:35:25,405 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:35:29,453 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:35:29,453 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:35:29,453 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:35:29,539 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:35:30,539 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:31,540 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:32,540 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:34,354 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:35:34,541 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:36,542 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:38,543 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:40,538 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:35:40,538 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:35:49,501 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:35:49,501 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:35:49,502 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:35:49,548 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:35:51,549 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:52,549 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:54,550 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:55,667 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:35:55,667 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:35:56,551 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:35:58,552 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:04,431 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:36:09,538 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:36:09,538 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:36:09,538 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:36:09,556 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:36:10,802 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:36:10,802 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:36:11,557 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:12,558 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:14,559 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:16,560 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:18,561 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:25,972 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:36:25,972 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:36:29,592 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:36:29,592 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:36:29,592 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:36:30,566 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:36:30,566 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:31,566 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:32,567 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:34,509 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:36:34,568 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:36,569 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:38,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:41,102 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:36:41,103 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:36:49,683 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:36:49,683 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:36:49,684 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:36:50,575 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:36:50,575 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:51,575 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:52,576 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:54,577 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:56,234 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:36:56,235 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:36:56,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:36:58,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:04,587 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:37:09,723 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:37:09,723 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:37:09,724 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:37:10,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:37:10,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:11,405 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:37:11,405 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:37:11,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:12,584 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:14,585 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:16,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:18,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:26,541 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:37:26,542 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:37:29,780 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:37:29,781 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:37:29,781 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:37:30,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:37:31,592 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:32,593 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:34,594 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:34,667 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:37:36,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:38,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:41,675 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:37:41,675 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:37:49,871 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:37:49,872 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:37:49,872 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:37:50,601 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:37:52,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:54,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:56,603 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:37:56,809 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:37:56,809 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:37:58,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:04,747 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:38:09,921 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:38:09,921 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:38:09,922 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:38:10,609 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:38:10,609 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:11,947 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:38:11,948 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:38:12,610 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:14,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:16,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:18,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:27,126 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:38:27,126 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:38:29,984 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:38:29,984 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:38:29,985 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:38:30,618 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:38:30,618 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:32,618 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:34,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:34,825 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:38:36,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:38,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:42,256 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:38:42,256 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:38:50,019 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:38:50,020 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:38:50,020 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:38:50,626 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:38:50,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:52,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:54,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:56,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:38:57,394 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:38:57,394 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:38:58,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:04,904 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:39:10,132 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:39:10,132 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:39:10,133 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:39:10,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:39:10,634 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:12,525 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:39:12,525 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:39:12,635 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:14,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:16,637 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:18,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:27,662 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:39:27,663 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:39:30,169 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:39:30,169 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:39:30,170 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:39:30,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:39:30,643 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:32,644 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:34,645 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:34,982 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:39:36,646 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:38,647 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:42,793 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:39:42,794 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:39:50,232 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:39:50,233 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:39:50,233 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:39:50,652 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:39:52,653 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:53,654 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:55,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:57,656 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:39:57,929 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:39:57,930 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:39:59,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:05,058 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:40:10,355 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:40:10,356 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:40:10,357 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:40:10,661 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:40:11,662 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:12,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:13,074 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:40:13,074 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:40:13,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:15,664 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:17,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:19,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:28,350 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:40:28,351 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:40:30,446 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:40:30,447 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:40:30,447 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:40:30,670 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:40:31,670 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:32,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:33,671 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:35,136 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:40:35,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:37,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:39,674 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:43,482 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:40:43,482 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:40:50,521 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:40:50,522 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:40:50,522 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:40:50,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:40:51,679 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:52,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:53,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:55,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:57,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:40:58,612 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:40:58,613 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:40:59,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:05,214 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:41:10,592 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:41:10,593 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:41:10,593 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:41:10,688 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:41:12,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:13,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:13,797 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:41:13,797 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:41:15,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:17,691 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:19,692 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:28,927 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:41:28,928 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:41:30,671 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:41:30,672 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:41:30,672 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:41:30,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:41:32,698 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:33,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:35,292 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:41:35,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:37,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:39,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:44,060 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:41:44,060 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:41:50,796 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:41:50,796 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:41:50,797 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:41:51,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:41:52,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:53,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:55,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:57,708 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:41:59,190 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:41:59,190 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:41:59,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:05,369 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:42:10,853 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:42:10,854 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:42:10,854 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:42:11,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:42:11,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:12,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:13,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:14,353 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:42:14,354 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:42:15,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:17,716 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:19,717 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:29,484 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:42:29,484 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:42:30,887 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:42:30,888 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:42:30,888 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:42:31,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:42:31,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:33,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:35,446 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:42:35,723 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:37,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:39,725 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:44,616 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:42:44,616 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:42:50,964 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:42:50,964 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:42:50,965 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:42:51,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:42:51,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:53,731 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:55,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:57,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:59,733 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:42:59,747 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:42:59,747 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:43:05,523 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:43:11,020 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:43:11,020 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:43:11,021 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:43:11,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:43:11,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:13,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:14,879 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:43:14,879 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:43:15,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:17,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:19,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:30,014 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:43:30,014 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:43:31,075 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:43:31,075 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:43:31,076 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:43:31,746 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:43:33,747 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:35,603 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:43:35,748 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:37,749 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:39,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:45,148 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:43:45,148 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:43:51,128 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:43:51,129 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:43:51,129 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:43:51,755 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:43:53,756 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:55,757 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:57,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:43:59,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:00,280 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:44:00,280 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:44:05,678 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:44:11,169 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:44:11,170 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:44:11,170 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:44:11,765 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:44:11,765 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:13,765 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:15,410 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:44:15,411 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:44:15,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:17,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:19,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:30,547 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:44:30,548 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:44:31,267 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:44:31,268 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:44:31,268 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:44:31,772 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:44:31,772 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:33,773 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:35,750 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:44:35,773 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:37,774 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:39,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:45,680 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:44:45,681 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:44:51,335 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:44:51,335 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:44:51,335 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:44:51,780 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:44:51,780 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:53,781 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:55,781 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:57,782 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:44:59,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:00,810 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:45:00,811 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:45:05,820 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:45:11,383 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:45:11,384 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:45:11,384 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:45:11,788 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:45:13,789 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:15,790 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:15,939 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:45:15,939 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:45:17,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:19,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:31,069 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:45:31,070 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:45:31,481 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:45:31,481 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:45:31,481 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:45:31,797 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:45:33,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:35,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:35,892 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:45:37,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:39,800 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:46,202 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:45:46,202 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:45:51,580 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:45:51,580 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:45:51,581 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:45:51,805 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:45:53,806 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:54,807 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:56,808 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:45:58,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:00,809 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:01,341 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:46:01,342 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:46:05,965 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:46:11,715 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:46:11,715 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:46:11,716 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:46:11,814 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:46:13,815 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:14,816 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:16,483 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:46:16,483 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:46:16,817 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:18,818 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:20,819 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:31,621 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:46:31,621 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:46:31,800 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:46:31,800 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:46:31,800 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:46:31,823 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:46:33,824 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:34,825 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:36,036 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:46:36,826 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:38,827 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:40,828 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:46,756 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:46:46,757 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:46:51,843 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:46:51,843 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:46:51,844 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:46:52,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:46:52,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:53,833 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:54,834 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:56,835 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:46:58,836 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:00,837 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:01,886 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:47:01,886 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:47:06,106 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:47:11,881 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:47:11,881 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:47:11,882 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:47:12,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:47:12,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:13,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:14,843 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:16,844 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:17,020 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:47:17,020 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:47:18,845 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:20,845 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:31,973 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:47:31,973 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:47:31,974 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:47:32,202 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:47:32,203 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:47:32,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:47:33,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:34,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:36,177 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:47:36,852 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:38,853 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:40,853 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:47,376 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:47:47,376 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:47:52,024 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:47:52,024 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:47:52,025 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:47:52,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:47:53,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:54,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:56,859 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:47:58,860 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:00,861 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:02,508 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:48:02,508 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:48:06,247 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:48:12,059 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:48:12,059 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:48:12,060 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:48:12,866 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:48:12,866 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:13,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:14,867 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:16,868 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:17,640 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:48:17,641 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:48:18,869 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:20,870 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:32,117 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:48:32,117 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:48:32,118 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:48:32,782 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:48:32,782 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:48:32,874 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:48:32,874 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:33,874 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:34,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:36,316 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:48:36,875 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:38,876 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:40,877 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:47,968 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:48:47,968 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:48:52,175 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:48:52,175 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:48:52,176 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:48:52,882 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:48:52,882 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:54,882 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:56,883 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:48:58,884 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:00,885 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:03,103 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:49:03,103 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:49:06,387 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:49:12,242 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:49:12,243 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:49:12,243 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:49:12,890 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:49:14,890 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:16,891 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:18,237 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:49:18,238 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:49:18,892 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:20,893 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:32,297 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:49:32,298 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:49:32,298 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:49:32,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:49:32,899 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:33,515 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:49:33,515 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:49:34,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:35,900 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:36,457 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:49:37,901 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:39,902 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:41,903 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:48,684 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:49:48,685 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:49:52,372 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:49:52,373 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:49:52,373 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:49:52,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:49:53,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:54,908 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:55,909 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:57,910 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:49:59,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:01,911 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:03,816 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:50:03,817 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:50:06,525 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:50:12,429 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:50:12,429 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:50:12,430 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:50:12,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:50:13,916 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:14,917 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:15,917 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:17,918 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:18,947 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:50:18,948 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:50:19,919 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:21,920 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:32,497 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:50:32,497 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:50:32,498 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:50:32,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:50:33,925 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:34,100 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:50:34,100 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:50:34,926 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:35,926 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:36,593 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:50:37,927 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:39,928 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:41,929 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:49,282 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:50:49,283 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:50:52,739 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:50:52,739 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:50:52,740 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:50:52,933 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:50:53,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:54,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:55,934 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:57,935 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:50:59,936 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:01,937 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:04,414 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:51:04,414 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:51:06,661 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:51:12,799 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:51:12,800 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:51:12,800 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:51:12,941 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:51:13,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:14,942 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:15,943 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:17,944 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:19,545 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:51:19,545 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:51:19,945 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:21,946 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:32,885 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:51:32,886 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:51:32,886 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:51:32,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:51:33,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:34,687 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:51:34,688 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:51:34,950 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:35,951 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:36,731 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:51:37,952 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:39,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:41,953 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:49,842 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:51:49,843 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:51:52,929 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:51:52,929 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:51:52,930 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:51:52,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:51:53,958 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:54,959 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:55,959 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:57,960 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:51:59,961 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:01,962 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:04,973 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:52:04,973 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:52:06,805 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:52:12,991 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:52:12,992 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:52:12,992 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:52:13,966 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:52:13,966 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:14,967 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:15,967 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:17,968 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:19,969 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:20,104 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:52:20,104 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:52:21,969 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:33,056 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:52:33,056 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:52:33,057 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:52:33,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:52:33,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:34,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:35,272 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:52:35,272 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:52:35,974 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:36,881 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:52:37,975 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:39,976 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:41,977 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:50,466 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:52:50,466 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:52:53,136 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:52:53,136 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:52:53,137 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:52:53,981 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:52:53,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:54,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:55,982 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:57,983 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:52:59,984 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:01,985 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:05,597 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:53:05,597 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:53:06,959 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:53:13,183 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:53:13,183 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:53:13,184 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:53:13,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:53:13,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:14,989 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:15,990 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:17,991 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:19,991 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:20,728 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:53:20,728 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:53:21,992 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:33,234 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:53:33,235 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:53:33,235 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:53:33,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:53:33,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:34,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:35,891 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:53:35,891 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:53:35,997 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:37,037 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:53:37,998 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:39,999 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:42,000 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:51,088 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:53:51,088 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:53:53,293 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:53:53,294 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:53:53,294 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:53:54,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:53:54,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:55,004 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:56,005 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:53:58,006 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:00,006 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:02,007 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:06,220 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:54:06,220 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:54:07,114 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:54:13,363 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:54:13,363 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:54:13,364 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:54:14,012 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:54:14,012 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:15,012 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:16,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:18,013 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:20,014 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:21,350 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:54:21,350 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:54:22,015 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:33,402 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:54:33,403 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:54:33,403 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:54:34,019 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:54:35,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:36,518 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:54:36,519 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:54:37,020 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:37,192 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:54:39,022 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:41,023 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:43,023 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:51,663 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:54:51,664 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:54:53,476 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:54:53,476 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:54:53,477 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:54:54,028 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:54:55,028 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:56,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:57,029 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:54:59,030 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:01,031 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:03,032 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:06,797 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:55:06,797 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:55:07,270 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:55:13,519 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:55:13,519 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:55:13,520 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:55:14,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:55:15,037 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:16,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:17,038 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:19,039 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:21,040 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:21,928 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:55:21,928 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:55:23,041 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:33,613 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:55:33,614 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:55:33,614 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:55:34,045 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:55:35,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:36,046 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:37,047 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:37,073 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:55:37,073 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:55:37,346 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:55:39,048 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:41,049 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:43,050 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:52,240 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:55:52,241 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:55:53,698 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:55:53,699 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:55:53,699 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:55:54,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:55:55,054 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:56,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:57,055 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:55:59,056 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:01,057 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:03,058 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:07,370 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:56:07,371 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:56:07,503 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:56:13,788 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:56:13,789 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:56:13,789 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:56:14,062 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:56:15,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:16,063 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:17,064 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:19,065 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:21,066 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:22,504 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:56:22,505 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:56:23,067 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:33,843 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:56:33,843 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:56:33,843 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:56:34,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:56:35,072 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:36,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:37,073 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:37,503 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:56:37,658 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:56:37,658 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:56:39,074 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:41,075 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:43,076 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:53,075 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:56:53,075 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:56:53,905 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:56:53,906 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:56:53,906 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:56:54,080 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:56:55,081 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:56,081 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:57,082 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:56:59,083 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:01,084 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:03,085 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:07,581 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:57:08,207 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:57:08,207 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:57:13,975 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:57:13,975 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:57:13,975 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:57:14,089 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:57:15,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:16,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:17,090 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:19,091 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:21,092 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:23,093 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:23,336 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:57:23,336 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:57:34,040 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:57:34,040 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:57:34,040 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:57:34,097 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:57:35,097 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:36,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:37,098 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:37,658 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:57:38,467 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:57:38,467 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:57:39,099 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:41,100 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:43,101 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:53,598 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:57:53,598 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:57:54,075 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:57:54,075 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:57:54,076 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:57:54,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:57:55,105 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:56,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:57,106 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:57:59,107 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:01,108 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:03,109 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:07,733 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:58:08,732 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:58:08,732 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:58:14,137 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:58:14,138 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:58:14,139 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:58:15,113 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:58:15,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:16,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:17,114 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:19,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:21,115 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:23,116 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:23,862 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:58:23,863 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:58:34,175 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:58:34,176 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:58:34,176 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:58:35,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:58:35,120 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:36,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:37,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:37,806 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:58:38,995 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:58:38,996 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:58:39,121 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:41,122 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:54,127 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:58:54,128 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:58:54,259 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:58:54,260 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:58:54,260 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:58:55,132 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:58:56,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:58:58,133 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:00,134 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:02,135 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:04,137 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:07,890 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:59:09,307 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:59:09,307 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:59:14,387 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:59:14,388 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:59:14,388 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:59:15,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:59:16,141 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:18,142 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:20,143 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:22,144 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:24,145 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:24,437 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:59:24,438 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:59:34,440 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:59:34,441 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:59:34,441 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:59:35,150 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:59:36,150 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:37,968 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 12:59:38,151 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:39,585 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:59:39,585 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:59:40,152 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:42,152 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:44,153 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:54,504 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 12:59:54,504 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 12:59:54,505 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 12:59:54,714 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 12:59:54,714 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 12:59:55,158 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 12:59:56,158 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 12:59:58,159 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:00,160 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:02,161 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:08,038 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:00:09,854 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:00:09,854 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:00:14,540 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:00:14,540 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:00:14,541 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:00:15,165 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:00:16,166 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:18,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:20,167 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:22,168 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:24,986 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:00:24,986 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:00:34,575 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:00:34,576 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:00:34,576 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:00:35,174 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:00:36,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:38,106 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:00:38,175 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:40,176 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:40,328 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:00:40,328 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:00:42,177 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:44,178 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:54,609 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:00:54,609 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:00:54,610 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:00:55,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:00:55,515 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:00:55,515 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:00:56,182 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:57,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:00:58,183 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:00,184 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:02,185 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:04,186 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:08,179 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:01:10,750 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:01:10,750 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:01:14,674 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:01:14,675 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:01:14,675 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:01:15,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:01:16,190 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:17,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:18,191 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:20,192 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:22,193 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:24,194 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:25,881 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:01:25,881 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:01:34,707 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:01:34,707 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:01:34,708 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:01:35,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:01:36,198 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:37,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:38,199 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:38,252 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:01:40,200 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:41,018 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:01:41,018 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:01:42,201 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:54,750 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:01:54,750 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:01:54,751 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:01:55,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:01:56,150 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:01:56,150 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:01:56,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:57,206 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:01:58,207 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:00,208 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:02,209 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:08,398 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:02:11,280 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:02:11,281 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:02:14,815 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:02:14,816 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:02:14,817 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:02:15,214 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:02:16,214 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:17,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:18,215 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:20,216 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:22,217 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:26,409 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:02:26,409 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:02:34,856 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:02:34,857 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:02:34,857 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:02:35,222 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:02:37,223 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:38,475 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:02:39,224 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:41,225 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:41,538 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:02:41,539 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:02:43,226 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:54,903 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:02:54,903 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:02:54,904 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:02:55,230 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:02:56,668 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:02:56,668 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:02:57,231 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:02:59,232 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:01,233 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:03,234 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:08,553 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:03:11,847 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:03:11,847 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:03:14,958 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:03:14,959 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:03:14,959 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:03:15,238 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:03:17,239 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:19,240 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:21,241 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:23,242 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:26,982 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:03:26,982 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:03:35,019 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:03:35,019 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:03:35,020 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:03:35,246 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:03:37,247 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:38,631 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:03:39,248 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:41,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:42,111 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:03:42,112 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:03:43,249 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:55,110 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:03:55,110 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:03:55,111 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:03:55,254 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:03:57,247 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:03:57,247 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:03:57,255 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:03:59,256 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:01,256 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:03,257 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:08,709 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:04:12,418 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:04:12,418 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:04:15,207 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:04:15,207 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:04:15,208 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:04:15,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:04:17,262 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:19,263 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:21,264 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:23,264 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:27,550 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:04:27,550 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:04:35,246 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:04:35,246 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:04:35,247 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:04:35,269 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:04:37,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:38,787 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:04:39,270 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:41,271 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:42,682 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:04:42,683 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:04:43,272 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:55,299 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:04:55,300 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:04:55,300 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:04:56,277 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:04:57,278 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:04:57,812 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:04:57,813 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:04:59,279 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:01,279 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:03,280 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:08,864 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:05:12,964 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:05:12,964 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:05:15,392 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:05:15,393 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:05:15,393 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:05:16,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:05:17,285 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:19,286 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:21,287 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:23,288 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:28,102 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:05:28,102 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:05:35,484 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:05:35,484 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:05:35,484 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:05:36,292 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:05:37,293 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:38,940 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:05:39,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:41,294 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:43,233 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:05:43,234 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:05:43,295 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:55,520 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:05:55,521 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:05:55,521 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:05:56,299 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:05:57,300 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:05:58,401 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:05:58,401 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:05:59,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:01,301 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:03,302 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:09,021 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:06:13,568 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:06:13,569 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:06:15,577 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:06:15,578 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:06:15,578 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:06:16,307 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:06:17,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:18,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:19,308 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:21,309 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:23,310 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:28,701 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:06:28,702 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:06:35,613 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:06:35,614 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:06:35,614 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:06:36,315 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:06:37,315 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:38,316 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:39,098 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:06:39,316 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:41,317 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:43,318 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:43,976 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:06:43,976 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:06:55,863 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:06:55,863 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:06:55,864 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:06:56,322 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:06:57,323 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:58,323 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:06:59,129 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:06:59,130 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:06:59,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:01,324 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:03,325 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:09,175 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:07:14,289 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:07:14,289 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:07:15,928 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:07:15,929 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:07:15,929 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:07:16,331 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:07:18,332 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:20,333 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:22,334 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:24,335 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:29,423 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:07:29,424 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:07:35,997 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:07:35,997 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:07:35,998 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:07:36,339 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:07:38,340 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:39,252 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:07:40,341 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:42,342 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:44,343 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:44,555 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:07:44,555 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:07:56,032 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:07:56,032 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:07:56,033 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:07:56,348 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:07:58,349 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:07:59,690 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:07:59,691 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:08:00,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:02,350 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:04,351 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:09,330 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:08:14,819 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:08:14,819 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:08:16,079 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:08:16,079 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:08:16,079 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:08:16,356 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:08:18,357 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:20,358 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:22,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:24,359 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:29,951 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:08:29,951 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:08:36,221 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:08:36,222 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:08:36,222 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:08:36,364 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:08:38,365 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:39,408 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:08:40,366 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:42,367 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:44,368 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:08:45,086 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:08:45,086 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:08:56,288 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:08:56,288 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:08:56,289 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:08:56,372 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:08:58,373 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:00,220 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:09:00,220 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:09:00,374 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:02,375 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:04,376 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:09,486 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:09:15,357 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:09:15,358 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:09:16,370 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:09:16,370 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:09:16,371 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:09:16,380 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:09:18,381 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:20,382 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:22,383 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:24,384 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:30,489 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:09:30,490 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:09:36,442 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:09:36,442 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:09:36,443 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:09:37,389 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:09:38,389 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:39,562 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:09:40,391 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:42,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:44,392 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:09:45,628 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:09:45,628 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:09:56,478 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:09:56,478 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:09:56,479 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:09:57,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:09:58,398 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:00,399 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:00,758 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:10:00,759 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:10:02,400 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:04,401 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:09,641 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:10:15,890 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:10:15,890 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:10:16,522 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:10:16,523 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:10:16,523 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:10:17,407 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:10:18,408 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:20,409 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:22,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:24,410 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:31,020 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:10:31,020 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:10:36,590 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:10:36,590 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:10:36,591 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:10:37,415 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:10:38,416 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:39,714 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:10:40,417 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:42,418 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:44,419 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:10:46,157 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:10:46,157 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:10:56,658 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:10:56,658 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:10:56,659 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:10:57,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:10:58,424 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:00,425 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:01,299 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:11:01,299 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:11:02,426 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:04,427 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:09,782 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:11:16,432 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:11:16,432 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:11:16,706 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:11:16,706 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:11:16,706 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:11:17,432 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:11:18,433 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:20,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:22,434 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:24,435 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:31,565 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:11:31,565 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:11:36,764 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:11:36,764 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:11:36,765 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:11:37,440 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:11:38,441 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:39,853 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:11:40,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:42,442 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:44,443 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:11:46,699 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:11:46,700 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:11:56,829 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:11:56,830 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:11:56,830 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:11:57,448 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:11:58,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:00,449 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:01,844 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:12:01,844 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:12:02,450 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:04,451 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:09,924 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:12:16,966 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:12:16,967 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:12:16,967 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:12:17,006 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:12:17,006 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:12:17,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:12:18,456 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:20,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:22,457 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:24,458 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:32,169 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:12:32,169 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:12:37,003 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:12:37,004 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:12:37,004 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:12:37,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:12:38,463 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:39,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:39,996 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:12:41,464 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:43,465 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:45,466 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:12:47,308 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:12:47,309 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:12:57,073 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:12:57,074 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:12:57,074 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:12:57,470 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:12:59,471 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:01,472 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:02,443 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:13:02,443 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:13:03,473 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:05,474 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:10,073 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:13:17,154 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:13:17,154 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:13:17,155 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:13:17,478 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:13:17,577 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:13:17,578 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:13:19,479 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:21,480 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:23,481 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:25,482 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:32,708 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:13:32,708 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:13:37,229 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:13:37,230 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:13:37,230 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:13:37,487 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:13:39,488 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:40,148 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:13:41,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:43,489 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:45,490 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:13:47,841 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:13:47,841 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:13:57,263 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:13:57,264 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:13:57,264 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:13:57,495 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:13:59,496 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:01,497 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:03,028 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:14:03,028 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:14:03,498 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:05,499 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:10,223 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:14:17,319 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:14:17,319 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:14:17,320 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:14:17,503 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:14:18,187 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:14:18,188 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:14:19,504 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:21,505 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:23,506 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:25,507 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:33,324 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:14:33,324 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:14:37,355 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:14:37,356 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:14:37,356 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:14:37,512 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:14:39,513 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:40,300 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:14:41,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:43,514 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:45,515 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:14:48,457 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:14:48,458 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:14:57,424 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:14:57,425 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:14:57,425 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:14:57,520 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:14:59,521 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:01,522 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:03,523 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:03,599 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:15:03,599 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:15:05,524 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:10,379 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:15:17,515 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:15:17,516 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:15:17,516 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:15:17,529 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:15:18,737 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:15:18,738 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:15:19,530 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:21,530 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:23,531 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:25,532 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:33,894 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:15:33,894 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:15:37,626 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:15:37,627 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:15:37,627 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:15:38,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:15:39,537 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:40,456 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:15:41,538 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:43,539 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:45,540 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:15:49,024 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:15:49,024 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:15:57,696 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:15:57,696 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:15:57,697 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:15:58,544 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:15:59,545 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:01,546 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:03,547 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:04,156 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:16:04,157 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:16:05,548 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:10,534 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:16:17,756 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:16:17,756 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:16:17,757 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:16:18,553 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:16:19,308 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:16:19,309 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:16:19,554 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:21,555 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:23,556 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:25,556 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:34,448 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:16:34,449 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:16:37,815 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:16:37,815 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:16:37,816 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:16:38,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:16:39,562 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:40,612 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:16:41,563 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:43,564 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:45,565 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:16:49,580 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:16:49,581 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:16:57,881 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:16:57,881 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:16:57,882 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:16:58,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:16:59,570 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:01,571 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:03,572 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:04,710 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:17:04,710 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:17:05,573 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:10,690 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:17:17,932 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:17:17,932 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:17:17,933 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:17:18,578 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:17:19,579 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:19,843 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:17:19,843 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:17:21,580 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:23,580 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:25,581 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:35,004 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:17:35,005 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:17:37,994 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:17:37,995 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:17:37,995 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:17:38,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:17:39,586 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:40,768 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:17:41,587 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:43,588 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:45,589 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:17:50,137 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:17:50,137 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:17:58,090 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:17:58,091 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:17:58,091 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:17:58,594 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:17:59,594 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:02,595 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:04,596 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:05,268 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:18:05,268 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:18:06,597 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:10,845 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:18:18,139 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:18:18,140 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:18:18,140 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:18:18,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:18:19,602 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:20,426 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:18:20,426 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:18:20,603 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:22,604 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:24,605 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:26,606 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:35,559 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:18:35,560 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:18:38,207 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:18:38,207 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:18:38,208 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:18:38,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:18:40,611 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:40,923 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:18:42,612 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:44,613 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:46,614 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:18:50,695 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:18:50,695 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:18:58,243 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:18:58,243 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:18:58,243 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:18:58,619 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:19:00,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:02,620 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:04,621 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:05,825 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:19:05,825 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:19:06,622 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:11,000 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:19:18,291 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:19:18,292 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:19:18,292 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:19:18,627 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:19:20,628 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:20,956 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:19:20,957 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:19:22,629 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:24,630 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:26,631 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:36,928 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:19:36,928 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:19:38,336 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:19:38,336 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:19:38,336 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:19:38,636 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:19:40,637 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:41,076 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:19:42,638 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:44,639 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:46,640 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:19:52,059 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:19:52,060 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:19:58,371 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:19:58,372 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:19:58,372 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:19:58,645 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:20:00,646 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:02,647 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:04,648 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:06,649 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:07,191 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:20:07,192 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:20:11,155 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:20:18,421 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:20:18,421 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:20:18,422 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:20:18,654 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:20:20,655 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:22,324 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:20:22,324 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:20:22,656 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:24,657 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:26,658 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:37,456 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:20:37,456 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:20:38,475 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:20:38,476 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:20:38,476 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:20:38,663 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:20:40,664 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:41,232 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:20:42,665 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:44,666 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:46,667 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:20:52,589 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:20:52,590 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:20:58,518 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:20:58,519 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:20:58,519 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:20:58,672 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:21:00,673 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:02,674 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:04,675 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:06,676 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:07,720 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:21:07,720 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:21:11,311 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:21:18,567 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:21:18,568 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:21:18,568 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:21:18,680 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:21:20,681 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:22,682 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:22,848 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:21:22,849 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:21:24,683 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:26,684 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:37,981 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:21:37,981 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:21:38,603 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:21:38,603 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:21:38,604 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:21:38,689 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:21:40,690 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:41,390 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:21:42,692 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:44,693 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:46,694 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:21:53,112 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:21:53,112 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:21:58,647 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:21:58,647 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:21:58,648 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:21:58,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:22:00,699 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:02,700 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:04,701 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:06,702 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:08,252 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:22:08,253 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:22:11,468 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:22:18,736 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:22:18,736 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:22:18,737 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:22:19,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:22:20,706 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:22,707 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:23,384 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:22:23,384 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:22:24,708 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:26,709 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:38,516 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:22:38,516 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:22:38,772 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:22:38,772 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:22:38,773 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:22:39,713 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:22:40,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:41,545 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:22:42,714 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:44,715 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:46,716 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:22:53,652 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:22:53,652 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:22:59,073 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:22:59,074 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:22:59,074 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:22:59,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:23:00,721 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:01,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:03,722 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:05,723 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:07,724 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:08,782 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:23:08,782 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:23:11,623 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:23:19,150 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:23:19,151 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:23:19,151 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:23:19,729 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:23:20,729 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:21,730 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:23,731 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:24,198 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:23:24,199 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:23:25,731 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:27,732 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:39,188 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:23:39,188 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:23:39,189 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:23:39,407 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:23:39,407 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:23:39,738 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:23:40,738 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:41,700 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:23:41,739 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:43,740 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:45,741 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:47,742 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:23:54,565 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:23:54,565 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:23:59,263 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:23:59,264 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:23:59,264 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:23:59,747 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:24:01,748 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:03,749 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:05,750 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:07,751 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:09,695 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:24:09,696 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:24:11,778 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:24:19,307 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:24:19,307 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:24:19,308 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:24:19,756 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:24:21,757 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:23,758 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:24,829 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:24:24,829 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:24:25,759 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:27,760 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:39,372 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:24:39,372 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:24:39,373 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:24:39,765 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:24:39,958 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:24:39,959 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:24:41,766 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:41,856 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:24:43,767 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:45,768 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:47,769 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:24:55,093 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:24:55,093 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:24:59,460 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:24:59,461 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:24:59,461 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:24:59,774 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:25:01,774 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:03,775 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:05,776 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:07,777 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:10,232 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:25:10,232 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:25:11,933 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:25:19,496 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:25:19,497 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:25:19,497 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:25:19,781 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:25:21,782 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:23,783 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:25,367 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:25:25,368 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:25:25,784 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:27,785 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:39,543 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:25:39,544 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:25:39,544 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:25:39,790 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:25:40,696 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:25:40,696 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:25:41,790 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:42,011 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:25:43,791 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:45,792 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:47,793 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:25:55,827 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:25:55,828 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:25:59,580 DEBUG SenderThread:656728 [sender.py:send():179] send: history +2021-07-15 13:25:59,581 DEBUG SenderThread:656728 [sender.py:send():179] send: summary +2021-07-15 13:25:59,581 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 13:25:59,798 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +2021-07-15 13:26:01,799 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:26:08,802 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:26:10,803 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:26:10,961 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:26:10,962 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status +2021-07-15 13:26:12,091 DEBUG SenderThread:656728 [sender.py:send():179] send: stats +2021-07-15 13:26:12,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log +2021-07-15 13:26:26,096 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 13:26:26,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status