diff --git "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" --- "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" +++ "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" @@ -4480,3 +4480,3959 @@ 2021-07-17 00:26:33,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log 2021-07-17 00:26:35,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log 2021-07-17 00:26:37,435 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:26:46,382 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:26:46,383 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:27:01,517 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:27:01,518 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:27:07,505 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:27:16,648 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:27:16,648 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:27:31,789 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:27:31,789 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:27:37,572 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:27:46,921 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:27:46,921 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:28:02,054 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:28:02,054 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:28:05,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:07,640 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:28:07,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:09,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:11,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:13,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:15,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:17,183 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:28:17,183 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:28:24,614 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:28:24,614 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:28:24,615 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:28:24,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:28:25,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:27,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:29,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:31,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:32,406 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:28:32,407 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:28:33,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:35,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:37,715 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:28:44,805 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:28:44,806 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:28:44,806 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:28:44,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:28:45,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:47,661 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:28:47,662 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:28:47,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:49,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:53,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:55,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:28:57,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:02,820 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:29:02,820 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:29:05,006 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:29:05,006 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:29:05,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:29:05,007 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:29:06,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:07,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:07,792 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:29:09,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:11,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:13,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:15,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:17,951 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:29:17,951 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:29:25,148 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:29:25,149 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:29:25,149 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:29:26,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:26,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:29:27,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:29,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:31,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:33,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:33,138 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:29:33,138 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:29:35,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:37,870 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:29:45,339 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:29:45,340 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:29:45,340 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:29:46,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:29:47,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:48,325 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:29:48,326 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:29:49,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:51,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:53,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:29:55,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:03,499 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:30:03,499 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:30:05,463 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:30:05,463 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:30:05,464 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:30:06,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:30:07,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:07,944 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:30:10,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:12,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:14,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:16,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:18,630 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:30:18,630 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:30:25,614 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:30:25,615 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:30:25,615 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:30:26,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:30:27,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:28,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:30,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:32,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:33,761 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:30:33,761 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:30:34,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:36,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:38,017 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:30:38,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:45,742 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:30:45,743 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:30:45,743 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:30:46,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:30:47,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:48,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:48,990 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:30:48,991 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:30:50,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:52,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:54,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:30:56,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:04,174 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:31:04,174 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:31:05,944 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:31:05,945 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:31:05,945 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:31:06,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:31:07,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:08,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:08,090 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:31:10,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:12,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:15,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:17,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:19,315 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:31:19,315 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:31:26,136 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:31:26,137 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:31:26,137 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:31:27,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:27,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:31:29,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:31,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:33,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:34,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:31:34,461 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:31:35,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:37,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:38,168 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:31:46,362 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:31:46,362 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:31:46,363 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:31:47,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:31:49,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:49,653 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:31:49,653 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:31:51,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:53,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:55,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:31:57,087 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:04,816 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:32:04,817 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:32:06,581 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:32:06,581 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:32:06,582 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:32:07,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:07,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:32:08,243 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:32:09,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:11,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:13,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:15,095 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:17,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:19,946 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:32:19,946 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:32:26,665 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:32:26,666 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:32:26,666 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:32:27,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:32:29,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:30,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:32,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:34,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:35,132 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:32:35,132 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:32:36,104 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:38,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:38,320 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:32:46,929 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:32:46,930 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:32:46,930 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:32:47,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:32:50,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:50,325 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:32:50,325 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:32:52,111 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:54,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:56,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:32:58,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:05,546 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:33:05,546 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:33:07,110 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:33:07,110 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:33:07,110 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:33:07,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:33:08,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:08,395 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:33:10,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:12,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:14,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:16,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:18,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:20,678 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:33:20,679 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:33:27,239 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:33:27,239 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:33:27,240 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:33:28,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:28,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:33:30,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:32,129 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:34,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:35,869 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:33:35,869 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:33:36,131 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:38,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:38,470 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:33:47,433 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:33:47,434 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:33:47,434 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:33:48,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:48,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:33:50,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:51,115 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:33:51,115 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:33:53,138 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:55,139 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:57,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:33:59,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:06,275 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:34:06,275 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:34:07,578 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:34:07,578 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:34:07,579 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:34:08,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:34:08,545 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:34:09,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:10,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:11,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:13,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:15,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:17,148 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:19,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:21,406 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:34:21,406 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:34:27,684 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:34:27,685 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:34:27,685 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:34:28,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:28,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:34:29,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:31,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:33,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:35,156 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:36,619 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:34:36,620 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:34:37,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:38,618 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:34:39,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:47,822 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:34:47,823 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:34:47,823 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:34:48,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:34:49,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:51,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:51,845 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:34:51,845 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:34:53,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:55,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:57,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:34:59,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:07,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:35:07,077 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:35:08,021 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:35:08,022 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:35:08,023 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:35:08,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:35:08,694 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:35:09,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:11,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:14,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:16,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:18,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:20,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:22,208 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:35:22,209 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:35:28,203 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:35:28,203 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:35:28,204 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:35:29,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:35:30,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:31,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:32,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:34,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:36,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:37,461 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:35:37,461 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:35:38,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:38,769 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:35:48,338 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:35:48,338 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:35:48,339 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:35:49,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:49,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:35:50,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:52,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:52,621 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:35:52,621 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:35:54,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:56,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:35:58,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:07,782 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:36:07,782 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:36:08,422 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:36:08,422 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:36:08,423 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:36:08,842 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:36:09,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:36:10,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:11,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:12,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:14,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:16,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:19,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:22,912 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:36:22,912 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:36:28,645 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:36:28,645 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:36:28,646 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:36:29,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:36:31,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:33,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:35,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:37,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:38,042 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:36:38,043 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:36:38,918 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:36:39,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:48,776 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:36:48,776 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:36:48,777 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:36:49,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:49,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:36:51,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:53,181 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:36:53,182 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:36:53,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:55,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:57,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:36:59,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:08,427 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:37:08,427 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:37:08,851 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:37:08,852 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:37:08,852 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:37:08,996 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:37:09,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:37:11,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:13,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:15,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:17,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:19,226 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:23,557 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:37:23,557 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:37:29,091 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:37:29,092 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:37:29,092 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:37:29,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:37:31,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:33,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:35,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:37,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:38,691 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:37:38,691 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:37:39,073 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:37:39,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:49,198 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:37:49,199 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:37:49,199 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:37:49,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:37:51,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:52,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:53,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:53,825 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:37:53,825 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:37:56,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:37:58,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:00,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:08,963 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:38:08,964 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:38:09,148 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:38:09,321 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:38:09,322 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:38:09,322 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:38:10,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:38:12,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:14,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:16,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:18,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:20,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:24,095 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:38:24,096 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:38:29,487 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:38:29,488 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:38:29,488 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:38:30,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:38:32,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:34,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:36,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:38,262 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:39,217 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:38:39,227 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:38:39,227 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:38:40,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:49,703 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:38:49,703 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:38:49,703 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:38:50,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:38:52,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:54,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:54,375 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:38:54,376 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:38:56,271 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:38:58,272 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:00,273 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:09,284 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:39:09,533 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:39:09,533 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:39:09,871 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:39:09,872 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:39:09,872 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:39:10,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:39:12,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:15,280 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:17,281 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:19,281 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:21,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:24,670 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:39:24,670 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:39:30,091 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:39:30,092 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:39:30,092 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:39:30,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:39:32,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:33,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:35,289 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:37,290 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:39,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:39,355 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:39:39,805 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:39:39,805 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:39:41,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:50,266 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:39:50,266 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:39:50,267 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:39:50,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:39:51,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:52,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:53,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:54,995 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:39:54,995 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:39:55,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:57,298 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:39:59,299 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:01,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:09,427 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:40:10,131 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:40:10,131 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:40:10,388 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:40:10,389 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:40:10,389 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:40:11,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:40:12,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:14,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:16,307 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:18,308 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:20,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:22,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:25,260 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:40:25,261 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:40:30,530 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:40:30,530 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:40:30,531 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:40:31,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:40:32,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:33,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:34,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:36,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:38,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:39,500 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:40:40,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:40,403 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:40:40,403 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:40:50,697 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:40:50,697 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:40:50,698 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:40:51,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:40:52,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:53,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:54,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:55,590 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:40:55,590 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:40:56,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:40:58,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:00,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:02,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:09,573 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:41:10,767 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:41:10,767 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:41:10,902 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:41:10,902 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:41:10,903 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:41:11,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:41:12,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:13,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:14,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:16,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:18,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:20,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:25,980 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:41:25,980 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:41:31,030 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:41:31,030 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:41:31,030 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:41:31,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:41:32,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:33,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:35,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:37,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:39,345 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:39,648 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:41:41,112 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:41:41,113 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:41:41,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:51,219 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:41:51,219 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:41:51,219 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:41:51,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:41:53,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:55,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:56,252 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:41:56,253 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:41:57,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:41:59,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:01,355 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:09,721 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:42:11,359 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:42:11,359 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:42:11,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:42:11,360 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:42:11,465 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:42:11,466 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:42:13,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:15,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:17,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:19,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:21,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:26,624 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:42:26,624 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:42:31,464 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:42:31,465 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:42:31,465 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:42:32,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:42:33,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:35,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:37,372 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:39,797 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:42:40,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:41,754 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:42:41,755 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:42:42,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:51,617 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:42:51,617 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:42:51,618 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:42:52,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:42:53,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:54,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:56,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:42:56,981 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:42:56,981 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:42:58,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:00,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:02,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:09,870 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:43:11,722 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:43:11,723 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:43:11,723 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:43:12,189 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:43:12,189 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:43:12,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:43:13,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:14,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:16,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:18,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:20,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:22,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:27,328 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:43:27,328 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:43:31,890 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:43:31,890 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:43:31,890 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:43:32,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:43:33,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:34,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:36,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:38,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:39,943 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:43:40,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:42,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:42,465 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:43:42,465 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:43:52,026 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:43:52,027 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:43:52,027 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:43:52,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:43:54,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:56,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:43:57,640 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:43:57,641 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:43:58,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:00,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:02,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:10,017 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:44:12,159 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:44:12,159 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:44:12,160 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:44:12,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:44:12,803 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:44:12,804 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:44:13,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:14,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:15,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:17,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:19,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:21,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:23,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:27,965 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:44:27,966 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:44:32,290 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:44:32,290 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:44:32,291 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:44:32,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:44:34,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:35,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:37,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:39,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:40,089 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:44:41,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:43,098 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:44:43,098 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:44:43,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:52,449 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:44:52,449 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:44:52,450 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:44:53,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:53,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:44:54,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:55,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:57,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:44:58,259 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:44:58,259 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:44:59,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:01,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:03,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:10,160 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:45:12,613 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:45:12,614 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:45:12,614 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:45:13,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:45:13,509 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:45:13,509 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:45:14,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:15,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:17,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:19,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:21,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:23,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:28,694 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:45:28,695 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:45:32,792 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:45:32,793 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:45:32,793 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:45:33,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:33,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:45:34,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:35,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:37,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:39,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:40,232 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:45:41,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:43,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:43,833 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:45:43,833 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:45:52,944 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:45:52,944 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:45:52,944 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:45:53,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:45:54,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:55,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:56,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:58,462 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:45:59,178 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:45:59,178 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:46:00,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:02,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:04,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:10,304 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:46:13,110 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:46:13,111 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:46:13,111 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:46:13,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:46:14,378 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:46:14,378 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:46:14,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:15,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:16,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:18,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:20,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:22,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:24,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:29,531 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:46:29,532 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:46:33,307 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:46:33,308 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:46:33,308 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:46:33,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:46:34,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:35,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:36,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:38,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:40,378 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:46:40,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:42,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:44,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:44,662 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:46:44,663 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:46:53,376 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:46:53,377 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:46:53,377 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:46:53,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:46:54,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:55,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:56,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:58,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:46:59,859 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:46:59,860 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:47:00,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:02,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:10,452 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:47:13,467 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:47:13,467 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:47:13,468 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:47:13,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:47:15,003 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:47:15,003 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:47:15,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:17,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:19,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:21,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:23,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:30,168 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:47:30,169 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:47:33,657 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:47:33,658 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:47:33,658 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:47:34,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:47:35,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:37,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:39,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:40,530 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:47:41,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:43,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:45,302 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:47:45,303 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:47:53,796 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:47:53,797 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:47:53,797 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:47:54,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:47:55,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:57,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:47:59,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:00,477 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:48:00,478 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:48:01,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:03,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:10,607 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:48:13,932 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:48:13,932 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:48:13,933 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:48:14,521 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:48:15,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:15,627 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:48:15,628 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:48:17,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:19,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:21,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:23,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:25,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:30,825 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:48:30,825 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:48:34,125 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:48:34,125 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:48:34,126 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:48:34,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:48:35,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:36,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:38,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:40,532 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:40,682 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:48:42,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:44,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:45,957 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:48:45,958 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:48:54,235 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:48:54,236 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:48:54,236 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:48:54,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:48:56,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:48:58,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:00,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:01,137 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:49:01,137 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:49:02,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:04,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:10,757 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:49:14,308 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:49:14,308 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:49:14,309 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:49:14,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:49:16,329 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:49:16,329 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:49:16,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:18,548 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:20,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:22,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:24,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:31,517 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:49:31,517 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:49:34,464 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:49:34,465 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:49:34,465 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:49:34,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:49:36,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:38,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:40,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:40,835 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:49:42,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:44,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:46,650 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:49:46,650 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:49:54,691 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:49:54,692 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:49:54,692 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:49:55,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:49:56,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:49:59,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:01,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:01,831 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:50:01,831 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:50:03,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:05,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:10,908 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:50:14,792 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:50:14,793 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:50:14,793 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:50:15,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:50:16,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:17,009 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:50:17,010 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:50:17,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:19,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:21,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:23,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:25,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:32,193 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:50:32,193 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:50:34,932 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:50:34,933 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:50:34,933 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:50:35,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:50:36,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:37,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:39,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:40,982 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:50:41,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:43,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:45,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:47,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:50:47,322 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:50:55,069 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:50:55,070 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:50:55,070 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:50:55,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:50:56,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:57,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:50:59,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:01,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:02,528 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:51:02,528 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:51:03,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:05,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:11,057 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:51:15,219 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:51:15,219 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:51:15,220 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:51:15,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:51:16,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:17,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:17,873 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:51:17,873 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:51:19,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:21,601 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:23,602 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:26,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:33,065 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:51:33,065 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:51:35,387 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:51:35,388 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:51:35,388 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:51:35,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:51:37,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:38,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:40,610 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:41,132 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:51:42,611 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:44,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:46,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:48,196 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:51:48,197 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:51:55,510 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:51:55,510 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:51:55,511 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:51:55,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:51:56,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:57,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:51:58,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:00,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:02,620 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:03,366 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:52:03,366 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:52:04,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:06,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:11,207 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:52:15,644 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:52:15,644 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:52:15,645 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:52:16,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:16,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:52:17,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:18,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:18,663 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:52:18,663 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:52:20,628 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:22,629 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:24,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:26,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:33,840 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:52:33,841 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:52:35,802 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:52:35,803 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:52:35,803 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:52:36,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:36,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:52:37,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:38,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:41,281 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:52:41,638 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:43,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:45,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:47,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:48,971 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:52:48,971 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:52:55,934 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:52:55,935 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:52:55,935 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:52:56,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:52:57,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:52:59,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:01,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:03,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:04,103 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:53:04,103 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:53:05,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:11,358 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:53:16,094 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:53:16,094 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:53:16,094 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:53:16,654 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:53:17,655 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:19,328 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:53:19,328 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:53:19,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:21,657 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:23,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:25,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:27,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:34,474 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:53:34,474 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:53:36,187 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:53:36,188 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:53:36,188 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:53:36,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:53:37,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:39,664 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:41,431 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:53:41,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:43,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:45,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:49,605 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:53:49,605 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:53:56,378 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:53:56,379 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:53:56,379 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:53:56,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:53:57,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:58,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:53:59,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:01,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:03,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:04,835 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:54:04,835 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:54:05,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:07,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:11,504 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:54:16,458 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:54:16,459 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:54:16,459 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:54:16,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:54:18,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:20,047 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:54:20,048 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:54:20,682 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:22,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:24,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:26,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:35,183 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:54:35,183 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:54:36,644 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:54:36,644 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:54:36,645 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:54:36,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:54:38,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:40,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:41,578 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:54:42,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:44,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:46,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:54:50,318 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:54:50,318 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:54:56,799 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:54:56,800 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:54:56,800 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:54:57,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:54:58,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:00,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:02,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:04,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:05,448 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:55:05,448 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:55:06,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:11,654 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:55:16,901 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:55:16,901 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:55:16,902 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:55:17,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:55:18,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:20,666 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:55:20,667 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:55:20,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:22,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:24,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:26,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:28,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:35,865 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:55:35,865 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:55:37,061 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:55:37,061 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:55:37,062 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:55:37,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:55:38,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:40,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:41,728 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:55:42,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:44,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:46,721 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:55:51,002 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:55:51,002 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:55:57,145 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:55:57,145 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:55:57,146 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:55:57,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:55:58,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:00,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:03,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:05,729 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:06,133 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:56:06,134 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:56:07,730 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:11,801 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:56:17,242 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:56:17,243 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:56:17,243 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:56:17,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:56:18,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:19,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:21,428 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:56:21,428 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:56:21,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:23,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:25,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:27,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:36,608 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:56:36,608 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:56:37,382 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:56:37,383 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:56:37,383 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:56:37,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:56:39,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:41,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:41,873 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:56:43,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:45,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:47,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:56:51,739 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:56:51,739 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:56:57,458 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:56:57,459 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:56:57,459 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:56:57,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:56:59,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:01,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:03,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:05,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:07,000 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:57:07,001 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:57:07,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:11,947 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:57:17,719 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:57:17,719 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:57:17,720 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:57:17,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:57:19,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:21,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:22,145 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:57:22,145 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:57:23,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:25,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:27,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:37,346 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:57:37,347 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:57:37,851 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:57:37,852 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:57:37,852 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:57:38,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:57:39,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:42,018 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:57:42,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:44,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:46,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:48,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:57:52,477 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:57:52,477 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:57:57,966 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:57:57,966 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:57:57,967 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:57:58,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:57:59,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:00,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:02,781 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:04,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:06,783 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:07,621 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:58:07,621 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:58:08,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:12,091 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:58:18,140 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:58:18,141 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:58:18,141 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:58:18,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:58:19,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:20,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:22,790 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:22,814 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:58:22,814 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:58:24,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:26,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:28,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:37,955 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:58:37,955 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:58:38,322 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:58:38,323 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:58:38,323 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:58:38,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:58:39,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:40,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:42,163 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:58:42,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:44,799 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:46,800 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:48,801 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:58:53,087 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:58:53,088 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:58:58,558 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:58:58,559 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:58:58,559 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:58:58,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:58:59,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:00,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:02,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:04,808 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:07,809 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:08,219 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:59:08,219 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:59:09,810 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:12,228 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:59:18,784 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:59:18,784 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:59:18,784 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:59:18,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:59:19,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:21,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:23,425 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:59:23,426 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:59:23,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:25,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:27,818 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:29,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:38,600 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:59:38,600 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:59:38,941 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:59:38,941 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:59:38,942 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:59:39,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 00:59:40,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:41,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:42,295 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 00:59:43,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:45,827 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:47,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:49,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 00:59:53,733 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 00:59:53,734 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 00:59:59,085 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 00:59:59,085 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 00:59:59,086 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 00:59:59,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:00:00,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:01,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:03,835 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:05,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:07,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:08,869 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:00:08,870 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:00:09,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:12,366 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:00:19,251 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:00:19,252 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:00:19,252 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:00:19,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:00:20,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:21,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:23,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:24,072 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:00:24,072 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:00:25,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:27,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:29,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:39,238 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:00:39,238 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:00:39,368 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:00:39,368 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:00:39,369 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:00:39,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:39,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:00:40,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:41,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:42,440 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:00:43,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:46,854 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:48,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:00:54,369 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:00:54,370 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:00:59,515 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:00:59,515 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:00:59,515 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:00:59,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:01:00,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:02,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:04,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:06,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:08,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:09,501 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:01:09,501 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:01:10,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:12,514 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:01:19,618 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:01:19,619 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:01:19,619 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:01:19,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:01:20,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:22,871 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:24,710 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:01:24,711 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:01:24,872 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:26,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:28,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:39,698 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:01:39,698 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:01:39,699 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:01:39,858 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:01:39,859 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:01:39,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:01:40,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:42,588 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:01:42,880 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:44,881 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:46,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:48,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:50,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:52,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:54,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:55,004 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:01:55,005 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:01:56,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:01:58,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:00,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:02,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:04,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:06,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:08,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:10,138 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:02:10,139 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:02:10,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:12,661 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:02:12,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:14,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:16,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:18,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:20,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:23,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:25,269 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:02:25,270 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:02:25,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:27,899 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:29,900 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:31,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:33,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:35,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:37,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:39,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:40,400 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:02:40,400 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:02:41,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:42,732 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:02:43,546 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:02:43,547 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:02:43,547 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:02:43,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:02:45,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:47,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:49,910 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:51,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:53,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:02:55,532 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:02:55,533 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:03:03,675 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:03:03,676 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:03:03,676 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:03:03,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:03:05,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:07,918 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:09,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:10,665 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:03:10,666 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:03:11,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:12,807 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:03:13,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:23,788 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:03:23,788 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:03:23,788 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:03:23,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:03:25,807 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:03:25,808 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:03:25,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:27,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:29,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:31,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:33,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:40,986 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:03:40,986 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:03:42,881 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:03:43,957 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:03:43,958 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:03:43,958 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:03:44,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:03:45,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:46,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:48,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:50,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:52,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:54,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:03:56,117 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:03:56,117 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:04:04,144 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:04:04,145 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:04:04,145 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:04:04,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:04:05,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:06,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:08,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:10,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:11,378 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:04:11,378 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:04:12,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:12,954 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:04:14,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:24,389 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:04:24,389 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:04:24,389 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:04:24,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:04:25,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:26,597 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:04:26,597 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:04:26,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:28,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:30,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:32,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:34,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:41,751 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:04:41,751 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:04:43,027 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:04:44,600 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:04:44,601 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:04:44,601 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:04:44,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:04:45,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:46,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:48,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:50,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:52,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:54,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:04:56,884 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:04:56,884 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:05:04,734 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:05:04,735 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:05:04,736 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:05:04,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:05:06,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:08,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:10,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:12,123 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:05:12,123 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:05:12,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:13,100 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:05:14,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:24,871 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:05:24,871 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:05:24,872 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:05:24,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:05:26,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:27,296 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:05:27,297 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:05:27,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:29,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:31,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:33,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:35,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:42,438 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:05:42,439 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:05:43,174 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:05:45,028 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:05:45,028 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:05:45,029 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:05:45,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:05:46,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:47,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:49,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:51,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:53,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:55,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:05:57,571 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:05:57,571 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:06:05,199 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:06:05,200 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:06:05,200 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:06:05,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:06:06,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:07,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:09,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:12,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:12,771 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:06:12,771 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:06:13,249 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:06:14,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:16,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:25,356 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:06:25,357 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:06:25,358 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:06:26,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:26,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:06:27,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:27,946 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:06:27,947 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:06:28,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:30,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:33,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:35,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:37,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:43,132 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:06:43,133 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:06:43,322 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:06:45,460 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:06:45,460 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:06:45,461 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:06:46,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:06:47,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:49,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:51,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:53,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:55,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:06:58,264 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:06:58,264 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:07:05,625 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:07:05,626 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:07:05,627 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:07:06,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:07:07,025 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:09,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:11,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:13,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:13,397 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:07:13,402 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:07:13,403 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:07:15,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:25,804 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:07:25,805 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:07:25,805 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:07:26,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:07:27,034 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:28,663 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:07:28,663 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:07:29,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:31,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:33,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:35,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:43,473 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:07:43,845 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:07:43,846 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:07:46,020 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:07:46,020 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:07:46,021 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:07:46,043 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:07:47,044 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:49,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:51,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:53,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:55,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:57,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:07:58,982 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:07:58,983 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:08:06,190 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:08:06,190 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:08:06,191 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:08:07,052 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:07,052 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:08:08,053 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:10,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:12,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:13,552 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:08:14,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:14,168 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:08:14,168 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:08:16,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:26,328 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:08:26,328 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:08:26,328 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:08:27,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:08:28,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:29,387 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:08:29,388 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:08:30,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:32,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:34,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:36,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:38,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:43,627 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:08:44,585 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:08:44,585 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:08:46,448 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:08:46,449 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:08:46,449 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:08:47,071 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:08:48,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:50,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:52,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:54,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:56,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:08:59,713 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:08:59,714 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:09:06,655 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:09:06,655 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:09:06,655 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:09:07,081 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:09:08,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:10,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:12,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:13,705 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:09:14,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:14,933 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:09:14,933 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:09:16,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:26,786 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:09:26,787 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:09:26,787 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:09:27,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:09:28,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:30,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:30,212 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:09:30,213 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:09:32,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:34,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:36,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:43,777 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:09:45,351 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:09:45,351 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:09:46,946 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:09:46,947 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:09:46,947 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:09:47,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:09:48,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:50,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:52,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:55,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:09:57,104 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:00,480 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:10:00,480 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:10:07,055 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:10:07,055 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:10:07,056 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:10:07,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:10:08,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:09,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:11,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:13,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:13,853 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:10:15,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:15,625 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:10:15,625 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:10:17,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:27,218 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:10:27,218 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:10:27,219 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:10:28,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:10:29,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:30,911 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:10:30,911 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:10:31,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:33,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:35,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:37,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:43,926 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:10:46,066 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:10:46,066 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:10:47,369 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:10:47,370 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:10:47,370 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:10:48,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:10:49,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:51,129 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:53,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:55,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:10:58,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:01,200 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:11:01,201 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:11:07,520 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:11:07,520 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:11:07,520 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:11:08,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:11:09,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:10,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:12,139 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:14,002 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:11:14,139 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:16,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:16,329 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:11:16,330 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:11:18,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:27,657 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:11:27,657 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:11:27,658 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:11:28,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:11:29,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:30,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:31,472 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:11:31,472 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:11:32,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:34,148 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:36,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:38,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:44,080 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:11:46,610 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:11:46,610 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:11:47,815 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:11:47,815 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:11:47,816 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:11:48,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:11:49,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:50,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:52,156 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:54,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:56,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:11:58,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:01,741 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:12:01,741 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:12:07,930 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:12:07,930 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:12:07,931 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:12:08,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:12:09,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:10,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:12,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:14,153 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:12:15,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:16,873 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:12:16,873 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:12:17,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:19,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:27,999 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:12:27,999 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:12:28,000 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:12:28,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:12:29,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:31,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:32,048 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:12:32,048 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:12:33,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:35,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:37,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:44,226 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:12:47,238 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:12:47,238 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:12:48,094 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:12:48,095 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:12:48,096 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:12:48,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:12:49,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:51,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:53,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:55,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:57,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:12:59,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:02,375 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:13:02,376 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:13:08,265 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:13:08,265 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:13:08,266 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:13:09,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:09,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:13:10,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:11,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:13,193 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:14,302 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:13:15,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:17,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:17,507 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:13:17,507 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:13:19,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:28,411 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:13:28,412 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:13:28,412 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:13:29,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:29,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:13:30,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:31,201 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:32,645 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:13:32,646 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:13:33,202 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:35,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:37,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:44,379 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:13:47,841 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:13:47,842 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:13:48,486 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:13:48,486 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:13:48,487 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:13:49,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:49,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:13:50,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:51,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:53,210 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:55,210 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:13:57,211 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:02,972 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:14:02,973 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:14:08,607 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:14:08,608 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:14:08,608 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:14:09,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:14:10,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:12,217 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:14,218 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:14,454 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:14:16,219 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:18,103 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:14:18,103 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:14:18,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:28,758 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:14:28,758 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:14:28,758 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:14:29,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:14:30,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:32,226 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:33,237 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:14:33,238 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:14:34,227 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:36,228 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:38,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:40,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:44,525 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:14:48,384 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:14:48,385 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:14:48,955 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:14:48,956 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:14:48,956 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:14:49,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:14:50,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:52,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:54,236 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:56,237 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:14:58,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:00,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:03,516 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:15:03,517 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:15:09,094 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:15:09,094 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:15:09,095 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:15:09,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:15:10,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:11,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:12,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:14,246 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:14,598 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:15:16,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:18,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:18,659 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:15:18,660 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:15:29,237 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:15:29,237 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:15:29,238 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:15:29,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:15:30,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:31,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:32,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:33,789 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:15:33,790 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:15:34,255 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:36,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:38,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:44,671 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:15:48,945 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:15:48,946 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:15:49,360 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:15:49,361 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:15:49,361 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:15:50,262 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:15:51,262 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:53,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:55,264 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:57,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:15:59,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:04,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:16:04,076 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:16:09,526 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:16:09,527 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:16:09,527 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:16:10,271 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:16:11,272 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:13,273 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:14,745 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:16:15,273 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:17,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:19,207 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:16:19,207 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:16:19,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:29,683 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:16:29,683 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:16:29,684 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:16:30,280 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:16:31,281 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:33,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:34,444 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:16:34,444 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:16:35,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:37,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:39,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:44,819 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:16:49,614 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:16:49,615 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:16:49,891 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:16:49,891 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:16:49,892 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:16:50,290 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:16:51,290 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:53,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:55,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:57,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:16:59,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:04,744 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:17:04,745 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:17:09,996 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:17:09,996 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:17:09,996 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:17:10,298 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:17:11,298 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:14,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:14,894 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:17:16,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:18,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:19,883 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:17:19,883 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:17:20,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:30,198 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:17:30,198 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:17:30,199 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:17:30,307 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:17:32,308 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:34,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:35,020 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:17:35,020 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:17:36,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:38,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:40,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:44,962 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:17:50,161 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:17:50,161 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:17:50,317 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:17:50,317 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:17:50,318 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:17:51,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:17:52,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:54,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:56,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:17:58,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:00,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:05,292 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:18:05,292 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:18:10,503 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:18:10,504 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:18:10,504 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:18:11,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:18:12,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:14,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:15,030 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:18:16,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:18,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:20,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:20,422 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:18:20,422 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:18:30,652 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:18:30,652 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:18:30,653 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:18:31,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:18:32,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:34,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:35,575 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:18:35,575 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:18:36,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:38,338 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:40,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:45,102 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:18:50,770 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:18:50,771 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:18:50,771 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:18:50,851 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:18:50,852 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:18:51,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:18:52,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:53,345 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:55,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:57,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:18:59,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:01,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:06,041 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:19:06,042 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:19:10,909 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:19:10,909 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:19:10,910 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:19:11,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:19:12,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:13,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:15,168 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:19:15,355 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:17,356 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:19,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:21,171 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:19:21,171 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:19:21,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:31,062 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:19:31,062 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:19:31,063 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:19:31,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:19:32,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:33,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:35,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:36,319 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:19:36,319 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:19:37,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:39,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:41,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:45,238 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:19:51,242 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:19:51,243 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:19:51,243 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:19:51,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:19:51,525 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:19:51,526 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:19:52,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:53,372 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:55,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:57,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:19:59,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:01,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:06,718 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:20:06,719 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:20:11,396 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:20:11,396 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:20:11,397 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:20:12,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:12,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:20:13,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:15,307 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:20:15,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:17,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:20,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:21,849 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:20:21,849 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:20:22,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:31,512 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:20:31,512 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:20:31,512 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:20:32,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:20:33,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:34,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:36,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:36,980 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:20:36,981 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:20:38,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:40,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:42,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:45,376 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:20:51,619 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:20:51,620 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:20:51,620 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:20:52,126 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:20:52,127 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:20:52,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:20:53,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:54,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:56,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:20:58,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:00,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:02,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:07,301 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:21:07,301 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:21:11,848 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:21:11,849 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:21:11,849 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:21:12,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:21:13,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:14,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:15,448 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:21:16,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:18,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:20,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:22,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:22,433 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:21:22,434 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:21:31,962 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:21:31,962 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:21:31,963 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:21:32,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:21:33,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:34,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:37,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:37,660 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:21:37,661 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:21:39,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:41,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:43,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:45,518 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:21:52,130 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:21:52,130 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:21:52,131 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:21:52,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:21:52,878 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:21:52,879 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:21:53,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:55,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:57,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:21:59,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:01,431 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:08,018 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:22:08,018 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:22:12,289 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:22:12,290 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:22:12,290 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:22:12,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:22:13,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:15,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:15,593 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:22:17,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:19,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:21,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:23,148 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:22:23,148 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:22:23,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:32,477 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:22:32,478 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:22:32,478 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:22:33,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:33,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:22:35,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:37,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:38,280 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:22:38,281 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:22:39,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:41,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:45,669 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:22:52,629 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:22:52,629 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:22:52,630 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:22:53,428 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:22:53,428 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:22:53,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:53,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:22:55,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:57,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:22:59,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:01,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:03,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:08,579 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:23:08,579 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:23:12,839 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:23:12,839 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:23:12,840 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:23:13,462 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:23:14,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:15,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:15,744 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:23:17,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:19,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:21,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:23,715 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:23:23,715 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:23:32,891 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:23:32,892 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:23:32,892 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:23:33,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:33,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:23:34,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:35,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:38,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:38,859 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:23:38,859 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:23:40,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:42,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:45,816 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:23:53,014 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:23:53,015 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:23:53,015 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:23:53,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:23:54,169 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:23:54,169 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:23:54,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:56,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:23:58,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:00,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:02,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:04,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:09,351 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:24:09,351 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:24:13,141 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:24:13,141 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:24:13,142 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:24:13,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:24:14,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:15,893 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:24:16,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:18,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:20,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:22,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:24,483 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:24:24,483 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:24:33,338 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:24:33,339 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:24:33,339 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:24:33,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:24:34,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:36,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:38,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:39,665 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:24:39,666 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:24:40,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:42,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:45,967 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:24:53,483 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:24:53,483 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:24:53,484 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:24:53,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:24:54,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:54,834 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:24:54,835 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:24:55,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:57,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:24:59,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:01,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:03,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:10,023 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:25:10,024 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:25:13,594 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:25:13,595 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:25:13,595 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:25:14,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:14,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:25:15,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:16,039 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:25:17,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:19,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:21,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:23,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:25,154 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:25:25,155 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:25:33,724 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:25:33,725 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:25:33,725 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:25:34,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:34,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:25:35,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:37,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:39,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:40,308 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:25:40,309 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:25:41,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:43,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:46,113 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:25:53,868 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:25:53,869 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:25:53,869 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:25:54,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:25:55,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:55,539 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:25:55,540 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:25:57,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:25:59,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:01,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:04,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:10,726 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:26:10,726 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:26:14,087 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:26:14,087 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:26:14,088 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:26:14,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:26:15,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:16,182 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:26:16,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:18,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:20,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:22,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:24,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:25,864 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:26:25,864 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:26:34,237 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:26:34,237 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:26:34,238 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:26:34,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:26:35,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:36,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:38,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:40,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:41,067 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:26:41,068 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:26:42,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:44,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:46,256 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:26:54,438 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:26:54,438 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:26:54,439 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:26:54,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:26:55,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:56,228 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:26:56,229 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:26:56,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:26:58,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:00,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:02,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:04,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:11,405 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:27:11,405 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:27:14,591 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:27:14,591 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:27:14,592 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:27:15,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:15,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:27:16,329 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:27:16,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:18,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:21,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:23,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:25,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:26,535 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:27:26,536 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:27:34,751 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:27:34,752 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:27:34,752 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:27:35,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:35,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:27:37,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:39,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:41,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:41,710 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:27:41,711 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:27:43,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:45,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:46,400 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:27:54,884 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:27:54,885 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:27:54,885 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:27:55,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:55,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:27:56,903 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:27:56,904 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:27:57,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:27:59,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:01,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:03,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:05,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:12,089 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:28:12,090 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:28:15,098 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:28:15,099 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:28:15,099 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:28:15,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:15,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:28:16,475 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:28:17,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:19,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:21,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:23,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:25,601 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:27,224 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:28:27,224 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:28:35,235 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:28:35,236 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:28:35,236 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:28:35,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:28:37,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:39,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:41,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:42,366 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:28:42,367 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:28:43,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:28:46,550 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:28:55,366 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:28:55,366 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:28:55,366 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:28:55,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:28:57,559 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:28:57,559 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:28:58,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:00,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:02,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:04,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:12,725 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:29:12,730 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:29:15,463 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:29:15,463 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:29:15,463 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:29:15,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:29:16,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:16,626 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:29:18,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:20,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:22,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:24,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:27,864 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:29:27,864 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:29:35,603 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:29:35,603 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:29:35,604 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:29:35,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:29:36,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:38,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:40,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:42,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:43,044 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:29:43,045 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:29:44,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:46,698 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:29:55,673 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:29:55,673 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:29:55,674 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:29:56,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:29:56,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:29:58,181 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:29:58,182 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:29:58,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:00,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:02,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:04,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:13,314 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:30:13,314 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:30:15,805 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:30:15,805 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:30:15,806 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:30:16,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:16,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:30:16,775 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:30:18,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:21,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:23,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:25,654 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:28,445 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:30:28,446 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:30:35,927 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:30:35,927 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:30:35,928 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:30:36,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:36,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:30:37,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:39,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:41,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:43,587 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:30:43,587 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:30:43,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:45,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:46,853 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:30:56,096 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:30:56,097 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:30:56,097 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:30:56,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:56,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:30:57,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:30:58,857 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:30:58,857 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:30:59,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:01,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:03,671 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:05,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:14,034 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:31:14,034 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:31:16,224 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:31:16,224 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:31:16,225 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:31:16,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:31:16,925 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:31:17,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:18,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:19,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:21,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:23,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:25,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:29,164 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:31:29,164 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:31:36,388 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:31:36,388 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:31:36,389 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:31:36,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:36,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:31:37,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:39,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:41,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:43,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:44,342 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:31:44,342 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:31:45,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:47,000 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:31:56,601 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:31:56,602 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:31:56,602 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:31:56,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:31:57,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:31:59,472 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:31:59,473 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:31:59,697 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:01,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:03,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:06,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:14,617 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:32:14,618 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:32:16,772 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:32:16,772 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:32:16,773 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:32:17,076 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:32:17,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:32:18,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:19,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:20,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:22,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:24,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:26,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:29,752 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:32:29,752 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:32:36,924 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:32:36,925 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:32:36,925 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:32:37,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:32:38,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:39,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:40,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:42,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:44,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:44,936 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:32:44,936 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:32:46,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:47,150 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:32:56,992 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:32:56,992 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:32:56,993 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:32:57,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:32:58,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:32:59,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:00,100 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:33:00,100 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:33:00,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:02,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:04,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:06,729 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:15,296 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:33:15,297 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:33:17,224 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:33:17,224 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:33:17,227 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:33:17,229 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:33:17,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:33:18,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:19,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:20,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:22,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:24,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:26,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:30,427 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:33:30,427 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:33:37,366 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:33:37,367 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:33:37,368 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:33:37,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:33:38,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:39,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:40,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:43,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:45,599 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:33:45,600 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:33:45,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:47,309 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:33:47,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:33:57,488 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:33:57,489 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:33:57,490 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:33:57,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:33:59,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:00,814 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:34:00,815 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:34:01,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:03,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:05,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:07,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:16,006 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:34:16,006 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:34:17,385 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:34:17,672 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:34:17,673 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:34:17,673 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:34:17,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:34:19,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:21,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:23,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:25,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:27,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:31,140 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:34:31,140 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:34:37,846 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:34:37,846 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:34:37,846 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:34:38,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:34:39,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:41,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:43,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:45,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:46,270 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:34:46,271 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:34:47,460 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:34:47,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:34:57,924 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:34:57,925 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:34:57,925 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:34:58,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:35:00,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:01,430 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:35:01,430 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:35:02,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:04,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:06,781 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:08,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:16,616 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:35:16,617 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:35:17,537 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:35:18,038 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:35:18,038 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:35:18,039 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:35:18,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:35:20,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:22,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:24,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:26,790 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:28,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:31,749 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:35:31,749 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:35:38,194 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:35:38,195 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:35:38,195 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:35:38,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:35:40,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:42,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:44,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:46,799 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:35:46,884 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:35:46,884 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:35:47,611 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:35:58,355 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:35:58,356 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:35:58,356 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:35:58,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:36:00,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:02,137 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:36:02,137 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:36:02,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:04,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:06,808 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:17,305 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:36:17,305 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:36:17,679 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:36:18,500 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:36:18,500 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:36:18,501 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:36:18,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:18,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:36:20,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:23,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:25,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:27,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:32,435 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:36:32,435 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:36:38,636 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:36:38,637 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:36:38,637 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:36:38,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:36:40,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:41,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:43,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:45,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:47,565 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:36:47,565 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:36:47,751 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:36:47,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:36:58,758 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:36:58,759 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:36:58,759 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:36:58,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:36:59,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:00,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:01,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:02,791 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:37:02,791 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:37:03,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:05,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:07,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:17,819 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:37:17,951 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:37:17,951 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:37:18,868 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:37:18,868 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:37:18,869 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:37:19,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:19,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:37:20,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:21,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:23,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:25,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:27,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:33,083 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:37:33,083 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:37:38,937 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:37:38,937 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:37:38,938 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:37:39,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:39,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:37:40,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:41,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:43,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:46,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:47,893 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:37:48,214 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:37:48,214 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:37:48,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:37:59,018 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:37:59,019 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:37:59,019 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:37:59,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:38:00,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:02,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:03,402 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:38:03,403 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:38:04,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:06,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:08,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:17,968 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:38:18,568 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:38:18,568 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:38:19,116 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:38:19,117 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:38:19,117 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:38:19,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:38:20,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:21,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:22,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:24,869 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:26,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:28,871 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:33,699 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:38:33,699 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:38:39,281 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:38:39,281 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:38:39,281 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:38:39,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:38:40,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:41,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:42,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:44,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:46,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:48,041 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:38:48,831 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:38:48,832 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:38:48,880 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:38:59,400 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:38:59,401 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:38:59,401 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:38:59,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:39:00,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:01,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:02,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:04,035 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:39:04,036 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:39:04,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:06,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:08,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:18,111 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:39:19,201 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:39:19,202 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:39:19,522 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:39:19,523 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:39:19,523 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:39:19,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:39:20,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:21,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:22,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:24,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:26,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:28,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:34,331 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:39:34,331 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:39:39,639 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:39:39,640 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:39:39,640 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:39:39,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:39:40,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:41,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:42,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:44,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:47,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:48,184 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:39:49,463 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:39:49,463 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:39:49,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:39:59,782 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:39:59,783 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:39:59,783 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:39:59,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:40:01,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:03,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:04,592 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:40:04,593 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:40:05,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:07,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:09,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:18,259 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:40:19,730 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:40:19,730 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:40:19,980 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:40:19,980 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:40:19,981 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:40:20,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:40:21,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:23,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:25,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:27,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:29,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:34,860 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:40:34,861 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:40:40,117 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:40:40,117 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:40:40,118 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:40:40,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:40:41,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:43,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:45,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:47,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:48,333 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:40:49,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:40:49,996 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:40:49,997 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:41:00,253 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:41:00,253 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:41:00,254 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:41:00,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:41:01,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:03,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:05,191 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:41:05,191 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:41:05,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:07,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:09,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:18,405 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:41:20,360 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:41:20,361 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:41:20,490 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:41:20,491 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:41:20,491 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:41:20,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:41:21,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:22,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:24,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:26,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:28,950 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:30,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:35,493 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:41:35,494 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:41:40,472 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:41:40,473 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:41:40,473 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:41:40,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:41:42,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:44,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:46,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:48,478 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:41:48,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:41:50,626 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:41:50,627 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:42:00,589 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:42:00,590 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:42:00,591 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:42:00,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:42:02,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:04,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:05,775 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:42:05,775 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:42:06,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:08,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:10,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:18,549 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:42:20,676 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:42:20,677 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:42:20,677 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:42:20,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:42:21,031 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:42:21,031 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:42:22,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:24,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:26,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:28,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:30,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:36,166 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:42:36,166 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:42:40,742 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:42:40,742 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:42:40,743 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:42:40,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:42:42,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:44,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:46,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:48,623 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:42:48,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:42:51,296 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:42:51,296 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:43:00,863 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:43:00,864 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:43:00,864 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:43:00,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:43:02,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:04,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:06,484 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:43:06,484 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:43:06,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:08,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:18,697 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:43:20,997 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:43:20,998 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:43:20,998 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:43:21,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:43:21,652 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:43:21,652 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:43:23,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:26,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:28,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:30,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:36,859 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:43:36,860 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:43:41,102 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:43:41,102 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:43:41,103 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:43:42,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:43:43,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:44,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:46,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:48,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:48,773 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:43:50,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:43:51,991 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:43:51,991 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:44:01,186 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:44:01,187 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:44:01,187 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:44:02,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:44:03,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:04,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:06,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:07,121 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:44:07,121 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:44:08,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:10,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:18,846 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:44:21,312 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:44:21,313 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:44:21,313 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:44:22,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:44:22,441 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:44:22,441 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:44:23,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:24,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:26,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:28,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:30,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:37,596 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:44:37,597 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:44:41,423 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:44:41,423 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:44:41,424 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:44:42,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:44:44,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:46,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:48,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:48,921 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:44:50,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:44:52,742 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:44:52,742 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:45:01,535 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:45:01,536 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:45:01,536 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:45:02,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:02,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:45:04,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:06,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:07,987 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:45:07,988 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:45:09,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:11,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:18,997 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:45:21,759 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:45:21,760 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:45:21,760 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:45:22,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:45:23,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:23,168 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:45:23,168 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:45:24,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:25,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:27,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:29,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:31,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:38,304 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:45:38,304 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:45:41,892 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:45:41,893 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:45:41,893 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:45:42,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:45:44,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:45,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:47,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:49,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:49,073 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:45:51,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:45:53,435 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:45:53,436 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:46:02,022 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:46:02,022 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:46:02,023 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:46:02,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:46:04,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:05,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:07,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:08,627 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:46:08,627 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:46:09,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:11,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:19,154 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:46:22,123 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:46:22,123 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:46:22,124 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:46:23,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:23,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:46:23,837 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:46:23,838 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:46:24,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:25,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:27,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:29,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:31,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:39,024 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:46:39,025 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:46:42,346 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:46:42,347 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:46:42,347 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:46:43,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:43,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:46:44,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:45,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:47,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:49,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:49,225 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:46:51,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:46:54,156 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:46:54,157 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:47:02,470 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:47:02,470 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:47:02,471 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:47:03,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:47:04,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:05,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:07,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:09,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:09,292 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:47:09,292 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:47:11,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:19,297 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:47:22,584 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:47:22,584 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:47:22,585 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:47:23,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:47:24,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:24,434 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:47:24,434 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:47:26,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:28,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:30,111 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:32,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:39,611 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:47:39,612 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:47:42,704 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:47:42,704 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:47:42,705 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:47:43,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:47:44,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:46,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:48,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:49,369 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:47:50,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:52,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:47:54,743 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:47:54,744 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:48:02,896 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:48:02,897 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:48:02,900 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:48:03,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:48:04,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:06,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:08,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:09,905 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:48:09,905 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:48:10,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:12,129 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:19,443 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:48:23,028 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:48:23,028 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:48:23,029 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:48:23,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:48:24,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:25,078 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:48:25,078 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:48:25,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:26,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:28,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:31,138 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:33,139 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:40,295 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:48:40,296 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:48:43,117 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:48:43,118 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:48:43,118 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:48:43,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:48:45,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:47,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:49,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:49,517 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:48:51,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:53,148 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:48:55,426 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:48:55,426 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:49:03,193 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:49:03,193 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:49:03,194 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:49:04,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:49:05,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:07,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:09,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:10,626 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:49:10,626 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:49:11,156 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:13,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:19,590 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:49:23,320 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:49:23,321 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:49:23,321 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:49:24,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:49:25,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:25,765 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:49:25,765 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:49:27,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:29,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:31,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:33,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:40,912 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:49:40,913 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:49:43,494 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:49:43,494 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:49:43,496 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:49:44,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:49:45,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:47,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:49,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:49,662 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:49:51,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:49:56,044 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:49:56,044 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:50:03,616 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:50:03,617 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:50:03,617 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:50:04,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:50:05,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:08,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:10,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:11,178 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:50:11,178 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:50:12,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:19,735 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:50:23,765 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:50:23,765 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:50:23,766 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:50:24,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:50:25,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:26,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:26,320 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:50:26,320 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:50:28,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:30,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:32,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:34,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:41,489 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:50:41,490 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:50:43,845 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:50:43,845 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:50:43,846 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:50:44,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:50:45,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:46,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:48,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:49,810 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:50:50,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:52,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:50:56,620 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:50:56,620 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:51:03,959 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:51:03,959 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:51:03,959 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:51:04,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:51:05,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:06,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:08,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:10,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:11,750 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:51:11,751 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:51:12,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:19,880 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:51:24,067 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:51:24,067 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:51:24,068 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:51:24,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:51:25,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:26,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:26,897 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:51:26,897 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:51:28,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:30,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:32,217 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:34,218 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:42,088 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:51:42,089 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:51:44,216 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:51:44,217 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:51:44,217 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:51:44,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:51:46,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:48,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:49,953 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:51:50,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:52,226 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:51:57,221 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:51:57,222 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:52:04,386 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:52:04,386 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:52:04,387 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:52:05,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:52:06,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:09,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:11,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:12,351 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:52:12,351 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:52:13,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:20,028 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:52:24,456 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:52:24,456 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:52:24,457 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:52:25,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:52:26,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:27,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:27,487 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:52:27,488 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:52:29,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:31,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:33,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:35,246 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:42,659 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:52:42,660 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:52:44,580 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:52:44,580 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:52:44,580 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:52:45,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:52:46,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:47,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:49,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:50,100 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:52:51,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:53,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:52:57,791 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:52:57,791 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:53:04,703 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:53:04,703 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:53:04,703 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:53:05,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:53:06,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:07,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:09,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:11,262 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:12,921 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:53:12,921 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:53:13,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:15,264 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:20,174 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:53:24,835 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:53:24,835 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:53:24,836 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:53:25,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:53:26,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:27,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:28,110 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:53:28,111 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:53:29,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:31,271 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:33,272 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:43,246 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:53:43,247 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:53:44,936 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:53:44,936 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:53:44,937 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:53:45,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:53:46,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:47,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:49,279 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:50,246 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:53:51,280 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:53,281 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:53:58,379 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:53:58,380 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:54:05,079 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:54:05,080 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:54:05,080 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:54:05,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:54:06,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:07,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:09,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:11,289 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:13,510 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:54:13,511 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:54:14,290 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:20,320 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:54:25,171 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:54:25,172 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:54:25,172 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:54:25,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:54:26,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:28,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:28,655 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:54:28,656 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:54:30,298 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:32,299 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:34,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:43,824 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:54:43,824 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:54:45,277 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:54:45,277 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:54:45,278 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:54:45,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:54:46,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:47,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:48,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:50,307 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:50,399 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:54:52,308 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:54,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:54:58,955 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:54:58,956 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:55:05,453 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:55:05,454 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:55:05,455 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:55:06,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:55:07,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:08,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:10,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:12,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:14,102 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:55:14,102 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:55:14,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:20,476 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:55:25,653 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:55:25,654 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:55:25,654 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:55:26,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:55:27,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:28,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:29,464 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:55:29,464 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:55:30,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:32,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:34,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:44,601 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:55:44,601 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:55:45,751 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:55:45,751 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:55:45,752 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:55:46,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:46,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:55:47,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:48,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:50,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:50,550 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:55:52,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:54,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:55:59,740 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:55:59,741 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:56:05,856 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:56:05,856 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:56:05,857 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:56:06,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:06,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:56:07,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:08,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:10,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:12,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:14,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:14,875 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:56:14,876 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:56:20,626 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:56:25,977 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:56:25,978 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:56:25,978 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:56:26,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:56:27,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:29,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:30,032 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:56:30,032 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:56:31,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:33,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:35,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:45,185 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:56:45,185 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:56:46,186 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:56:46,186 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:56:46,187 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:56:46,356 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:56:47,356 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:49,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:50,703 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:56:51,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:53,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:56:55,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:00,314 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:57:00,314 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:57:06,279 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:57:06,279 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:57:06,280 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:57:06,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:57:07,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:09,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:11,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:13,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:15,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:15,446 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:57:15,446 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:57:20,773 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:57:26,404 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:57:26,405 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:57:26,405 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:57:27,372 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:27,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:57:29,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:30,598 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:57:30,599 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:57:31,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:33,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:35,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:45,737 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:57:45,737 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:57:46,468 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:57:46,469 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:57:46,469 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:57:47,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:47,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:57:49,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:50,846 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:57:52,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:54,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:57:56,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:00,867 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:58:00,868 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:58:06,552 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:58:06,552 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:58:06,553 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:58:07,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:58:08,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:10,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:12,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:14,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:15,999 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:58:15,999 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:58:16,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:20,922 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:58:26,676 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:58:26,676 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:58:26,677 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:58:27,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:58:28,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:30,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:31,136 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:58:31,137 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:58:32,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:34,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:36,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:46,317 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:58:46,317 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:58:46,744 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:58:46,744 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:58:46,745 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:58:47,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:58:48,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:50,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:50,999 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:58:52,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:54,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:58:56,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:01,449 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:59:01,450 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:59:06,878 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:59:06,879 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:59:06,879 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:59:07,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:59:08,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:10,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:12,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:14,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:16,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:16,588 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:59:16,589 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:59:21,074 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:59:26,976 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:59:26,976 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:59:26,977 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:59:27,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:59:28,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:30,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:31,766 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:59:31,767 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:59:32,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:34,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:36,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:46,913 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 01:59:46,914 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 01:59:47,059 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 01:59:47,060 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 01:59:47,060 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 01:59:47,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 01:59:48,433 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:50,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:51,150 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 01:59:52,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 01:59:54,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:02,044 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:00:02,044 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:00:07,139 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:00:07,140 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:00:07,140 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:00:07,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:00:08,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:09,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:11,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:13,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:15,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:17,175 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:00:17,176 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:00:21,227 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:00:27,282 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:00:27,282 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:00:27,283 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:00:27,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:00:29,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:31,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:32,339 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:00:32,340 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:00:33,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:35,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:37,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:47,416 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:00:47,416 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:00:47,417 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:00:47,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:00:47,568 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:00:47,568 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:00:49,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:51,302 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:00:51,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:53,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:55,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:00:57,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:02,718 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:01:02,719 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:01:07,592 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:01:07,593 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:01:07,593 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:01:08,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:01:09,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:11,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:13,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:15,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:17,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:17,850 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:01:17,850 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:01:21,378 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:01:27,677 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:01:27,678 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:01:27,678 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:01:28,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:01:29,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:31,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:33,354 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:01:33,355 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:01:33,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:35,477 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:47,758 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:01:47,759 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:01:47,759 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:01:48,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:01:48,528 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:01:48,529 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:01:49,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:51,452 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:01:51,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:53,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:01:55,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:03,682 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:02:03,682 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:02:07,863 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:02:07,864 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:02:07,864 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:02:08,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:02:09,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:10,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:12,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:14,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:16,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:18,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:18,813 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:02:18,814 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:02:21,525 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:02:28,060 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:02:28,061 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:02:28,061 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:02:28,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:02:29,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:30,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:32,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:34,004 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:02:34,005 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:02:34,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:36,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:48,183 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:02:48,184 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:02:48,184 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:02:48,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:02:49,244 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:02:49,245 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:02:49,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:50,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:51,598 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:02:52,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:54,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:56,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:02:58,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:04,396 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:03:04,397 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:03:08,278 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:03:08,279 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:03:08,280 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:03:08,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:03:09,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:10,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:12,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:14,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:16,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:19,532 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:03:19,532 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:03:21,674 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:03:28,437 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:03:28,438 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:03:28,438 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:03:28,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:03:30,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:32,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:34,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:34,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:03:34,695 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:03:37,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:39,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:48,617 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:03:48,617 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:03:48,618 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:03:49,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:49,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:03:49,870 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:03:49,870 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:03:50,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:51,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:51,753 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:03:53,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:55,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:03:57,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:05,043 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:04:05,044 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:04:08,727 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:04:08,728 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:04:08,728 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:04:09,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:04:10,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:11,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:13,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:15,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:17,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:20,173 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:04:20,174 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:04:21,828 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:04:28,861 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:04:28,862 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:04:28,862 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:04:29,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:04:30,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:31,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:33,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:35,343 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:04:35,344 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:04:35,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:37,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:48,980 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:04:48,981 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:04:48,981 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:04:49,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:04:50,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:50,607 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:04:50,608 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:04:51,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:51,898 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:04:53,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:55,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:04:57,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:05,783 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:05:05,784 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:05:09,093 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:05:09,094 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:05:09,094 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:05:09,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:09,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:05:10,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:11,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:13,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:15,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:18,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:20,914 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:05:20,914 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:05:21,969 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:05:29,182 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:05:29,182 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:05:29,183 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:05:29,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:05:30,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:32,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:34,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:36,051 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:05:36,052 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:05:36,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:38,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:49,281 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:05:49,281 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:05:49,282 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:05:49,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:05:50,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:51,258 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:05:51,258 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:05:52,044 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:05:52,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:54,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:56,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:05:58,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:06,432 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:06:06,432 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:06:09,383 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:06:09,383 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:06:09,384 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:06:09,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:06:10,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:12,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:14,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:16,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:18,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:21,564 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:06:21,564 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:06:22,118 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:06:29,556 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:06:29,557 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:06:29,557 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:06:29,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:06:30,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:32,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:34,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:36,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:36,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:06:36,695 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:06:38,601 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:49,714 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:06:49,715 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:06:49,715 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:06:50,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:50,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:06:51,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:51,875 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:06:51,875 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:06:52,187 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:06:52,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:54,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:56,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:06:58,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:07,010 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:07:07,010 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:07:09,788 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:07:09,788 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:07:09,789 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:07:10,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:07:11,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:13,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:15,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:17,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:19,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:22,150 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:07:22,151 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:07:22,281 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:07:29,920 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:07:29,920 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:07:29,921 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:07:30,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:07:31,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:33,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:35,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:37,287 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:07:37,287 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:07:37,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:39,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:50,034 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:07:50,034 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:07:50,035 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:07:50,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:07:51,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:52,322 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:07:52,537 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:07:52,537 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:07:53,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:55,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:57,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:07:59,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:07,669 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:08:07,669 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:08:10,189 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:08:10,190 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:08:10,191 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:08:10,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:08:11,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:13,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:15,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:17,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:19,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:22,387 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:08:22,798 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:08:22,798 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:08:30,331 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:08:30,331 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:08:30,332 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:08:30,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:08:31,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:33,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:36,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:37,929 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:08:37,929 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:08:38,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:40,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:50,405 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:08:50,405 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:08:50,406 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:08:50,655 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:08:51,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:52,454 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:08:52,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:53,129 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:08:53,129 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:08:54,657 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:56,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:08:58,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:00,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:08,329 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:09:08,330 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:09:10,525 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:09:10,525 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:09:10,526 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:09:10,664 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:09:12,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:14,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:16,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:18,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:22,522 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:09:23,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:09:23,461 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:09:30,578 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:09:30,578 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:09:30,579 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:09:30,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:09:32,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:34,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:36,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:38,594 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:09:38,594 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:09:38,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:40,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:50,676 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:09:50,676 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:09:50,677 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:09:50,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:09:52,588 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:09:52,682 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:53,726 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:09:53,727 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:09:54,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:56,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:09:58,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:00,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:02,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:04,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:07,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:08,859 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:10:08,859 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:10:09,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:11,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:13,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:15,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:17,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:19,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:21,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:22,657 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:10:23,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:23,992 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:10:23,992 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:10:25,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:27,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:29,697 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:31,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:33,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:35,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:37,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:39,126 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:10:39,127 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:10:39,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:41,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:43,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:45,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:47,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:49,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:51,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:10:52,729 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 02:10:54,263 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:10:54,263 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 02:10:54,673 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 02:10:54,674 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 02:10:54,674 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 02:10:54,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 02:11:02,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:11:05,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:11:07,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 02:11:09,394 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 02:11:09,394 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status