diff --git "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" @@ -0,0 +1,4016 @@ +2021-07-16 09:59:22,666 INFO MainThread:783720 [internal.py:wandb_internal():88] W&B internal server running at pid: 783720, started at: 2021-07-16 09:59:22.666048 +2021-07-16 09:59:22,668 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: check_version +2021-07-16 09:59:22,668 INFO WriterThread:783720 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/run-13hxxunp.wandb +2021-07-16 09:59:22,669 DEBUG SenderThread:783720 [sender.py:send():179] send: header +2021-07-16 09:59:22,669 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: check_version +2021-07-16 09:59:22,709 DEBUG SenderThread:783720 [sender.py:send():179] send: run +2021-07-16 09:59:22,884 INFO SenderThread:783720 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files +2021-07-16 09:59:22,884 INFO SenderThread:783720 [sender.py:_start_run_threads():716] run started: 13hxxunp with start time 1626429562 +2021-07-16 09:59:22,884 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 09:59:22,884 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:59:22,885 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: run_start +2021-07-16 09:59:23,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 09:59:24,007 DEBUG HandlerThread:783720 [meta.py:__init__():39] meta init +2021-07-16 09:59:24,007 DEBUG HandlerThread:783720 [meta.py:__init__():53] meta init done +2021-07-16 09:59:24,007 DEBUG HandlerThread:783720 [meta.py:probe():210] probe +2021-07-16 09:59:24,008 DEBUG HandlerThread:783720 [meta.py:_setup_git():200] setup git +2021-07-16 09:59:24,044 DEBUG HandlerThread:783720 [meta.py:_setup_git():207] setup git done +2021-07-16 09:59:24,044 DEBUG HandlerThread:783720 [meta.py:_save_pip():57] save pip +2021-07-16 09:59:24,045 DEBUG HandlerThread:783720 [meta.py:_save_pip():71] save pip done +2021-07-16 09:59:24,045 DEBUG HandlerThread:783720 [meta.py:probe():252] probe done +2021-07-16 09:59:24,049 DEBUG SenderThread:783720 [sender.py:send():179] send: files +2021-07-16 09:59:24,049 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-16 09:59:24,056 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:59:24,056 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:59:24,186 DEBUG SenderThread:783720 [sender.py:send():179] send: config +2021-07-16 09:59:24,186 DEBUG SenderThread:783720 [sender.py:send():179] send: config +2021-07-16 09:59:24,186 DEBUG SenderThread:783720 [sender.py:send():179] send: config +2021-07-16 09:59:24,524 INFO Thread-11 :783720 [upload_job.py:push():137] Uploaded file /tmp/tmpfr55ayyqwandb/lkng81xp-wandb-metadata.json +2021-07-16 09:59:24,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-metadata.json +2021-07-16 09:59:24,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/requirements.txt +2021-07-16 09:59:24,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 09:59:39,205 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:59:39,206 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:59:39,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 09:59:41,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 09:59:52,094 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 09:59:53,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/config.yaml +2021-07-16 09:59:54,347 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:59:54,348 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:00:09,480 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:00:09,480 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:00:22,175 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:00:24,612 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:00:24,613 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:00:39,745 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:00:39,745 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:00:52,255 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:00:54,876 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:00:54,876 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:01:10,009 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:01:10,009 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:01:22,327 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:01:25,145 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:01:25,146 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:01:40,288 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:01:40,288 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:01:52,402 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:01:55,475 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:01:55,475 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:02:10,610 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:02:10,611 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:02:22,475 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:02:25,756 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:02:25,756 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:02:40,888 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:02:40,888 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:02:52,550 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:02:56,021 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:02:56,022 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:03:11,156 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:03:11,156 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:03:22,628 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:03:26,298 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:03:26,299 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:03:41,433 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:03:41,433 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:03:52,705 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:03:56,568 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:03:56,568 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:04:11,700 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:04:11,701 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:04:22,782 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:04:26,836 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:04:26,837 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:04:41,973 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:04:41,974 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:04:52,859 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:04:57,109 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:04:57,109 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:05:12,242 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:05:12,243 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:05:16,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:05:22,938 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:05:27,393 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:05:27,393 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:05:42,547 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:05:42,547 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:05:53,017 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:05:57,680 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:05:57,680 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:06:12,812 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:06:12,813 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:06:23,094 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:06:27,952 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:06:27,953 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:06:43,087 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:06:43,088 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:06:53,172 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:06:58,221 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:06:58,222 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:07:13,359 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:07:13,360 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:07:20,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:07:23,253 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:07:28,511 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:07:28,511 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:07:43,679 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:07:43,680 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:07:53,334 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:07:58,834 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:07:58,834 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:08:13,967 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:08:13,967 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:08:23,415 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:08:29,106 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:08:29,106 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:08:44,236 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:08:44,239 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:08:53,493 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:08:59,376 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:08:59,377 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:09:14,509 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:09:14,509 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:09:23,589 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:09:29,652 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:09:29,652 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:09:33,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:35,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:37,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:39,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:44,784 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:09:44,784 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:09:50,386 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:09:50,386 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:09:50,387 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:09:51,138 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:09:53,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:53,673 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:09:55,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:57,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:59,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:09:59,915 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:09:59,916 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:10:10,423 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:10:10,423 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:10:10,424 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:10:11,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:10:11,147 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:13,147 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:15,063 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:10:15,064 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:10:15,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:17,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:19,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:23,751 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:10:30,212 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:10:30,212 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:10:30,460 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:10:30,461 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:10:30,461 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:10:31,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:10:31,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:33,155 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:35,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:37,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:39,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:45,346 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:10:45,346 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:10:50,498 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:10:50,499 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:10:50,499 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:10:51,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:10:51,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:53,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:53,828 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:10:55,164 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:57,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:10:59,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:00,489 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:11:00,489 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:11:10,536 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:11:10,537 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:11:10,537 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:11:11,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:11:13,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:15,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:15,622 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:11:15,622 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:11:17,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:19,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:23,906 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:11:30,572 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:11:30,573 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:11:30,573 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:11:30,755 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:11:30,755 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:11:31,178 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:11:33,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:35,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:37,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:39,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:45,922 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:11:45,923 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:11:50,607 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:11:50,608 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:11:50,608 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:11:51,185 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:11:51,186 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:53,186 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:53,984 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:11:55,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:57,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:11:59,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:01,054 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:12:01,054 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:12:10,645 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:12:10,645 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:12:10,645 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:12:11,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:12:11,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:13,194 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:15,194 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:16,187 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:12:16,187 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:12:17,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:19,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:24,062 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:12:30,681 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:12:30,681 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:12:30,682 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:12:31,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:12:31,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:31,350 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:12:31,350 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:12:33,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:35,202 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:37,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:39,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:46,661 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:12:46,661 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:12:50,719 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:12:50,719 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:12:50,720 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:12:51,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:12:51,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:53,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:54,140 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:12:55,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:57,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:12:59,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:01,979 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:13:01,979 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:13:10,757 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:13:10,757 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:13:10,758 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:13:11,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:13:11,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:13,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:15,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:17,112 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:13:17,112 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:13:17,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:19,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:24,219 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:13:30,793 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:13:30,793 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:13:30,794 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:13:31,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:13:31,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:32,258 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:13:32,258 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:13:33,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:35,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:37,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:39,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:47,426 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:13:47,426 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:13:50,828 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:13:50,829 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:13:50,829 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:13:51,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:13:53,234 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:54,296 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:13:55,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:57,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:13:59,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:02,559 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:14:02,559 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:14:10,863 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:14:10,863 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:14:10,864 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:14:11,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:14:13,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:15,243 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:17,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:17,692 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:14:17,692 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:14:19,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:24,373 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:14:30,899 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:14:30,899 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:14:30,900 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:14:31,249 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:14:32,844 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:14:32,845 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:14:33,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:34,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:36,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:38,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:40,253 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:47,984 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:14:47,984 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:14:50,936 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:14:50,936 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:14:50,937 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:14:51,258 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:14:53,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:54,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:54,451 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:14:56,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:14:58,261 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:00,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:03,115 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:15:03,115 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:15:10,972 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:15:10,972 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:15:10,972 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:15:11,267 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:15:12,267 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:13,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:14,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:16,269 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:18,246 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:15:18,246 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:15:18,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:20,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:24,529 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:15:31,009 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:15:31,009 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:15:31,010 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:15:31,276 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:15:32,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:33,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:33,406 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:15:33,406 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:15:34,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:36,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:38,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:40,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:48,544 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:15:48,544 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:15:51,044 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:15:51,045 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:15:51,045 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:15:51,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:15:52,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:53,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:54,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:54,606 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:15:56,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:15:58,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:00,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:03,978 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:16:03,978 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:16:11,082 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:16:11,083 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:16:11,084 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:16:11,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:16:12,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:13,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:14,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:16,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:18,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:19,118 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:16:19,118 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:16:20,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:24,684 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:16:31,121 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:16:31,121 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:16:31,122 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:16:31,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:16:32,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:33,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:34,250 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:16:34,251 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:16:34,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:36,307 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:38,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:40,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:49,384 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:16:49,384 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:16:51,157 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:16:51,158 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:16:51,158 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:16:51,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:16:52,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:53,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:54,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:54,763 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:16:56,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:16:58,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:00,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:04,516 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:17:04,516 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:17:11,195 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:17:11,196 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:17:11,196 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:17:11,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:17:13,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:14,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:16,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:18,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:19,647 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:17:19,648 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:17:20,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:24,841 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:17:31,237 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:17:31,237 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:17:31,238 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:17:31,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:17:33,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:34,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:34,781 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:17:34,781 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:17:36,335 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:38,336 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:40,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:49,932 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:17:49,932 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:17:51,274 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:17:51,274 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:17:51,274 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:17:51,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:17:53,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:54,343 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:54,918 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:17:56,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:17:58,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:00,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:05,187 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:18:05,187 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:18:11,310 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:18:11,311 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:18:11,311 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:18:11,350 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:18:13,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:14,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:16,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:18,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:20,319 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:18:20,319 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:18:20,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:24,995 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:18:31,349 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:18:31,349 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:18:31,350 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:18:31,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:18:33,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:34,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:35,459 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:18:35,459 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:18:36,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:38,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:40,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:50,590 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:18:50,590 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:18:51,385 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:18:51,385 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:18:51,386 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:18:52,368 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:18:52,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:53,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:54,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:55,075 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:18:56,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:18:58,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:00,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:05,720 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:19:05,721 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:19:11,422 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:19:11,422 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:19:11,423 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:19:12,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:19:12,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:13,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:14,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:16,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:18,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:20,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:20,858 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:19:20,859 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:19:25,152 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:19:31,456 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:19:31,456 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:19:31,457 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:19:32,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:19:32,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:33,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:34,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:35,991 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:19:35,991 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:19:36,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:38,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:40,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:51,124 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:19:51,125 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:19:51,492 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:19:51,492 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:19:51,493 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:19:52,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:19:52,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:54,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:55,228 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:19:56,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:19:58,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:00,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:06,255 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:20:06,255 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:20:11,527 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:20:11,528 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:20:11,528 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:20:12,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:20:12,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:14,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:16,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:18,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:20,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:21,386 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:20:21,386 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:20:25,305 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:20:31,562 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:20:31,563 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:20:31,563 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:20:32,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:20:32,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:34,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:36,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:36,518 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:20:36,518 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:20:38,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:40,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:51,600 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:20:51,600 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:20:51,601 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:20:51,658 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:20:51,658 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:20:52,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:20:54,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:55,383 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:20:56,420 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:20:58,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:00,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:06,828 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:21:06,828 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:21:11,638 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:21:11,638 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:21:11,639 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:21:12,426 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:21:14,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:16,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:18,429 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:20,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:21,966 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:21:21,967 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:21:25,460 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:21:31,676 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:21:31,676 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:21:31,677 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:21:32,435 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:21:34,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:36,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:37,100 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:21:37,100 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:21:38,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:40,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:51,710 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:21:51,711 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:21:51,711 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:21:52,258 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:21:52,259 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:21:52,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:21:52,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:54,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:55,538 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:21:56,444 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:21:58,445 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:00,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:07,394 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:22:07,394 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:22:11,747 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:22:11,748 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:22:11,748 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:22:12,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:22:12,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:14,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:16,453 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:18,453 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:20,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:22,525 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:22:22,525 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:22:25,617 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:22:31,781 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:22:31,781 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:22:31,782 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:22:32,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:22:32,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:34,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:36,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:37,654 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:22:37,655 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:22:38,461 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:40,462 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:51,817 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:22:51,817 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:22:51,817 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:22:52,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:22:52,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:52,789 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:22:52,790 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:22:54,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:55,693 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:22:56,469 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:22:58,470 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:00,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:07,956 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:23:07,957 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:23:11,851 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:23:11,852 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:23:11,852 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:23:12,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:23:12,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:14,477 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:16,477 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:18,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:20,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:23,087 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:23:23,088 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:23:25,772 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:23:31,885 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:23:31,885 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:23:31,886 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:23:32,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:23:32,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:34,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:36,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:38,219 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:23:38,219 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:23:38,487 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:40,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:51,920 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:23:51,920 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:23:51,921 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:23:52,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:23:53,374 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:23:53,374 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:23:54,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:55,849 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:23:56,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:23:58,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:00,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:08,509 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:24:08,510 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:24:11,955 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:24:11,955 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:24:11,956 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:24:12,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:24:14,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:16,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:18,508 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:20,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:23,639 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:24:23,640 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:24:25,928 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:24:31,990 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:24:31,990 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:24:31,991 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:24:32,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:24:32,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:34,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:36,516 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:38,517 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:38,778 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:24:38,778 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:24:40,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:52,024 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:24:52,025 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:24:52,025 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:24:52,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:24:52,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:53,939 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:24:53,940 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:24:54,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:56,007 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:24:56,525 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:24:58,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:00,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:09,078 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:25:09,079 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:25:12,061 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:25:12,061 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:25:12,062 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:25:12,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:25:12,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:14,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:16,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:18,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:20,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:24,208 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:25:24,209 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:25:26,085 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:25:32,096 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:25:32,096 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:25:32,096 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:25:32,540 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:25:32,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:34,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:36,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:38,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:39,341 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:25:39,342 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:25:40,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:52,130 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:25:52,131 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:25:52,131 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:25:52,549 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:25:52,549 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:54,475 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:25:54,475 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:25:54,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:56,164 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:25:56,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:25:58,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:00,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:09,632 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:26:09,632 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:26:12,167 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:26:12,167 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:26:12,168 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:26:12,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:26:12,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:14,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:16,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:18,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:20,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:24,762 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:26:24,762 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:26:26,243 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:26:32,205 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:26:32,205 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:26:32,205 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:26:32,566 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:26:34,567 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:36,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:38,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:39,894 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:26:39,895 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:26:41,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:52,247 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:26:52,248 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:26:52,248 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:26:52,575 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:26:54,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:55,028 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:26:55,028 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:26:55,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:56,321 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:26:57,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:26:59,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:01,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:10,160 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:27:10,160 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:27:12,281 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:27:12,282 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:27:12,282 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:27:12,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:27:14,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:15,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:17,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:19,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:21,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:25,293 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:27:25,293 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:27:26,400 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:27:32,321 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:27:32,321 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:27:32,322 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:27:32,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:27:33,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:34,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:35,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:37,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:39,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:40,424 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:27:40,425 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:27:41,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:52,358 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:27:52,358 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:27:52,359 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:27:52,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:27:53,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:54,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:55,559 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:27:55,559 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:27:55,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:56,478 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:27:57,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:27:59,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:01,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:10,690 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:28:10,690 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:28:12,393 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:28:12,394 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:28:12,394 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:28:12,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:28:13,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:14,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:15,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:17,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:19,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:21,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:25,825 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:28:25,825 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:28:26,554 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:28:32,429 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:28:32,430 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:28:32,430 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:28:32,624 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:28:33,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:34,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:35,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:37,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:39,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:40,955 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:28:40,956 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:28:41,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:52,467 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:28:52,468 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:28:52,468 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:28:52,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:28:53,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:54,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:55,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:56,085 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:28:56,086 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:28:56,632 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:28:57,628 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:28:59,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:01,630 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:11,217 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:29:11,217 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:29:12,506 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:29:12,506 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:29:12,506 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:29:12,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:29:13,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:14,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:15,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:17,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:19,637 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:21,638 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:26,349 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:29:26,349 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:29:26,708 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:29:32,543 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:29:32,543 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:29:32,544 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:29:32,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:29:33,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:34,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:35,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:41,480 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:29:41,480 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:29:53,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:55,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:56,624 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:29:56,624 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:29:56,787 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:29:57,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:29:59,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:01,654 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:03,655 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:05,656 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:07,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:09,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:11,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:11,757 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:30:11,758 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:30:13,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:15,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:17,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:19,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:21,663 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:23,663 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:25,664 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:26,863 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:30:26,892 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:30:26,893 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:30:27,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:29,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:31,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:33,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:35,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:37,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:39,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:41,671 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:42,026 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:30:42,026 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:30:43,672 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:45,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:47,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:49,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:51,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:54,557 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:30:54,557 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:30:54,558 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:30:54,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:30:55,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:56,942 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:30:57,156 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:30:57,157 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:30:57,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:30:59,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:01,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:12,289 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:31:12,290 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:31:14,607 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:31:14,607 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:31:14,608 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:31:14,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:31:15,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:17,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:19,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:21,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:27,020 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:31:27,419 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:31:27,420 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:31:34,645 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:31:34,646 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:31:34,646 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:31:34,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:31:35,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:37,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:40,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:42,552 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:31:42,552 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:31:42,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:54,682 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:31:54,683 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:31:54,683 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:31:54,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:31:56,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:57,099 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:31:57,688 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:31:57,689 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:31:57,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:31:58,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:00,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:02,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:04,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:12,822 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:32:12,823 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:32:14,721 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:32:14,721 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:32:14,721 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:32:15,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:32:16,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:17,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:18,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:20,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:22,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:24,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:27,176 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:32:27,954 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:32:27,954 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:32:34,761 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:32:34,761 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:32:34,762 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:32:35,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:32:36,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:37,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:38,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:40,719 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:42,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:43,089 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:32:43,089 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:32:44,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:54,798 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:32:54,799 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:32:54,799 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:32:55,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:32:55,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:56,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:32:57,254 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:32:58,221 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:32:58,221 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:32:58,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:00,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:02,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:04,730 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:13,369 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:33:13,370 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:33:14,836 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:33:14,837 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:33:14,837 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:33:15,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:33:15,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:16,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:18,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:20,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:22,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:27,333 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:33:28,503 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:33:28,503 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:33:34,876 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:33:34,877 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:33:34,877 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:33:35,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:33:35,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:36,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:38,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:40,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:42,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:43,632 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:33:43,633 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:33:54,913 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:33:54,914 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:33:54,914 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:33:55,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:33:55,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:56,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:57,408 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:33:58,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:33:58,762 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:33:58,763 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:34:00,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:02,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:04,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:13,894 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:34:13,895 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:34:14,952 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:34:14,952 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:34:14,953 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:34:15,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:34:15,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:16,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:18,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:20,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:22,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:24,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:27,487 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:34:29,028 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:34:29,028 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:34:34,992 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:34:34,992 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:34:34,992 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:34:35,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:34:35,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:36,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:38,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:40,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:42,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:44,157 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:34:44,157 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:34:44,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:55,028 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:34:55,029 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:34:55,029 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:34:55,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:34:55,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:56,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:57,564 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:34:58,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:34:59,288 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:34:59,289 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:35:00,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:02,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:04,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:14,418 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:35:14,418 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:35:15,067 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:35:15,068 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:35:15,068 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:35:15,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:35:15,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:16,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:18,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:20,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:22,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:24,791 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:27,641 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:35:29,988 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:35:29,988 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:35:35,107 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:35:35,108 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:35:35,108 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:35:35,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:35:35,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:36,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:38,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:40,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:42,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:45,125 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:35:45,125 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:35:55,147 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:35:55,147 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:35:55,148 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:35:55,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:35:56,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:35:57,719 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:35:58,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:00,256 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:36:00,256 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:36:00,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:02,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:15,186 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:36:15,186 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:36:15,187 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:36:15,419 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:36:15,419 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:36:15,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:36:16,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:18,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:20,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:22,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:27,797 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:36:30,560 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:36:30,560 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:36:35,226 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:36:35,226 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:36:35,227 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:36:35,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:36:36,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:38,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:40,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:42,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:45,689 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:36:45,690 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:36:55,265 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:36:55,265 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:36:55,266 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:36:55,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:36:56,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:36:57,874 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:36:58,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:00,820 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:37:00,821 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:37:00,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:02,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:15,301 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:37:15,302 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:37:15,302 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:37:15,838 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:37:15,973 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:37:15,973 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:37:16,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:18,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:20,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:22,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:24,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:27,952 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:37:31,111 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:37:31,111 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:37:35,340 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:37:35,340 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:37:35,341 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:37:35,847 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:37:36,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:38,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:40,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:42,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:44,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:46,240 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:37:46,241 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:37:55,378 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:37:55,379 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:37:55,379 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:37:55,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:37:56,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:37:58,030 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:37:58,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:00,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:01,370 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:38:01,370 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:38:02,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:04,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:15,423 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:38:15,423 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:38:15,424 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:38:15,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:38:16,536 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:38:16,537 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:38:16,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:18,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:20,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:22,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:24,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:28,107 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:38:31,673 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:38:31,674 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:38:35,463 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:38:35,463 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:38:35,463 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:38:35,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:38:36,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:38,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:40,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:42,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:44,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:46,807 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:38:46,807 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:38:55,500 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:38:55,501 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:38:55,501 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:38:55,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:38:56,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:38:58,182 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:38:58,884 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:00,885 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:01,984 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:39:01,984 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:39:02,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:04,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:15,538 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:39:15,539 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:39:15,539 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:39:15,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:39:16,892 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:17,145 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:39:17,145 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:39:18,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:20,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:22,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:24,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:28,251 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:39:32,277 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:39:32,278 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:39:35,576 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:39:35,577 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:39:35,577 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:39:35,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:39:36,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:38,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:40,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:42,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:47,405 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:39:47,406 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:39:55,614 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:39:55,614 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:39:55,615 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:39:55,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:39:56,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:39:58,320 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:39:58,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:00,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:02,544 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:40:02,544 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:40:02,911 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:04,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:15,652 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:40:15,652 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:40:15,652 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:40:15,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:40:16,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:17,683 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:40:17,683 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:40:18,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:20,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:22,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:24,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:28,390 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:40:32,820 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:40:32,821 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:40:35,688 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:40:35,689 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:40:35,689 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:40:35,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:40:36,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:38,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:40,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:42,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:44,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:47,953 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:40:47,954 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:40:55,724 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:40:55,725 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:40:55,725 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:40:55,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:40:56,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:40:58,459 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:40:58,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:00,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:02,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:03,083 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:41:03,084 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:41:04,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:15,762 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:41:15,762 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:41:15,763 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:41:15,942 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:41:16,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:18,219 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:41:18,219 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:41:18,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:20,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:22,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:24,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:28,532 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:41:33,358 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:41:33,358 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:41:35,802 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:41:35,802 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:41:35,803 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:41:35,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:41:36,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:38,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:40,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:42,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:44,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:48,498 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:41:48,498 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:41:55,842 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:41:55,842 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:41:55,842 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:41:55,959 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:41:56,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:41:58,601 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:41:58,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:00,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:02,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:03,630 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:42:03,631 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:42:15,880 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:42:15,880 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:42:15,881 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:42:15,967 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:42:16,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:18,780 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:42:18,780 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:42:18,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:20,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:22,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:28,672 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:42:33,914 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:42:33,914 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:42:35,922 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:42:35,922 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:42:35,923 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:42:35,975 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:42:36,976 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:38,976 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:40,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:42,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:49,044 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:42:49,045 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:42:55,960 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:42:55,960 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:42:55,961 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:42:55,984 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:42:56,984 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:57,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:42:58,741 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:42:59,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:01,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:03,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:04,190 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:43:04,191 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:43:15,995 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:43:15,995 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:43:15,996 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:43:16,992 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:43:16,993 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:17,993 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:19,320 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:43:19,321 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:43:19,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:21,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:23,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:25,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:28,811 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:43:34,454 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:43:34,455 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:43:36,032 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:43:36,032 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:43:36,032 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:43:37,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:43:37,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:38,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:40,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:42,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:44,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:46,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:49,586 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:43:49,587 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:43:56,068 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:43:56,068 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:43:56,068 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:43:57,011 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:43:57,011 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:58,011 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:43:58,879 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:44:00,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:02,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:04,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:04,717 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:44:04,717 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:44:06,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:16,107 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:44:16,107 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:44:16,108 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:44:17,019 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:44:17,019 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:18,020 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:19,848 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:44:19,849 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:44:20,020 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:22,021 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:24,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:26,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:28,951 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:44:34,981 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:44:34,981 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:44:36,149 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:44:36,149 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:44:36,150 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:44:37,028 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:44:37,029 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:38,029 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:40,030 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:42,030 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:44,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:46,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:50,119 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:44:50,120 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:44:56,189 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:44:56,189 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:44:56,190 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:44:57,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:44:57,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:58,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:44:59,024 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:45:00,038 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:02,039 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:04,039 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:05,266 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:45:05,266 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:45:06,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:16,229 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:45:16,229 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:45:16,229 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:45:17,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:45:17,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:18,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:20,046 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:20,398 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:45:20,398 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:45:22,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:24,048 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:29,095 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:45:35,528 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:45:35,529 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:45:36,267 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:45:36,267 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:45:36,267 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:45:37,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:45:37,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:38,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:40,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:42,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:44,057 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:50,659 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:45:50,659 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:45:56,309 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:45:56,309 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:45:56,310 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:45:57,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:45:57,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:58,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:45:59,167 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:46:00,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:02,065 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:04,066 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:05,793 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:46:05,793 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:46:16,348 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:46:16,348 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:46:16,349 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:46:17,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:46:18,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:20,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:20,923 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:46:20,923 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:46:22,074 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:24,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:29,239 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:46:36,077 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:46:36,077 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:46:36,386 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:46:36,386 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:46:36,386 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:46:37,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:46:38,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:40,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:42,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:44,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:51,207 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:46:51,208 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:46:56,422 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:46:56,422 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:46:56,423 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:46:57,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:46:58,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:46:59,313 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:47:00,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:02,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:04,092 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:06,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:06,354 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:47:06,354 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:47:16,459 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:47:16,459 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:47:16,459 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:47:17,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:47:18,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:20,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:21,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:47:21,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:47:22,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:24,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:26,102 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:29,391 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:47:36,498 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:47:36,498 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:47:36,499 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:47:36,620 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:47:36,620 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:47:37,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:47:38,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:40,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:42,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:44,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:46,110 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:51,753 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:47:51,753 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:47:56,536 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:47:56,537 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:47:56,537 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:47:57,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:47:58,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:47:59,470 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:48:00,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:02,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:04,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:06,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:06,892 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:48:06,892 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:48:16,573 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:48:16,574 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:48:16,574 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:48:17,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:48:18,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:20,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:22,023 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:48:22,024 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:48:22,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:24,126 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:26,127 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:29,548 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:48:36,610 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:48:36,610 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:48:36,611 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:48:37,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:48:37,165 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:48:37,165 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:48:38,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:40,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:42,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:44,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:46,135 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:52,337 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:48:52,337 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:48:56,647 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:48:56,647 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:48:56,648 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:48:57,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:48:58,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:48:59,626 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:49:00,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:02,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:04,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:07,468 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:49:07,468 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:49:16,684 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:49:16,684 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:49:16,685 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:49:17,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:49:18,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:20,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:22,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:22,599 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:49:22,599 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:49:24,152 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:29,703 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:49:36,723 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:49:36,724 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:49:36,724 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:49:37,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:49:37,778 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:49:37,779 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:49:38,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:40,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:42,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:44,161 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:52,915 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:49:52,916 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:49:56,764 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:49:56,764 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:49:56,764 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:49:57,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:49:58,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:49:59,961 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:50:00,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:02,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:04,169 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:08,278 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:50:08,278 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:50:16,799 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:50:16,799 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:50:16,800 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:50:17,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:50:18,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:20,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:22,176 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:23,411 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:50:23,411 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:50:24,177 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:30,039 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:50:36,835 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:50:36,836 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:50:36,836 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:50:37,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:50:38,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:38,549 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:50:38,550 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:50:40,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:42,184 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:44,185 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:50:53,686 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:50:53,686 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:50:56,874 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:50:56,874 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:50:56,875 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:50:57,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:50:58,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:00,115 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:51:00,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:02,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:04,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:06,194 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:08,817 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:51:08,817 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:51:16,911 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:51:16,911 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:51:16,911 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:51:17,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:51:18,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:20,200 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:22,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:23,947 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:51:23,948 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:51:24,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:30,194 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:51:36,948 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:51:36,949 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:51:36,949 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:51:37,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:51:38,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:39,087 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:51:39,088 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:51:40,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:42,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:44,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:51:54,222 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:51:54,222 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:51:56,992 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:51:56,992 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:51:56,993 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:51:57,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:51:58,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:00,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:00,271 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:52:02,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:04,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:09,350 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:52:09,351 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:52:17,029 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:52:17,030 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:52:17,030 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:52:17,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:52:18,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:19,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:21,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:23,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:24,481 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:52:24,481 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:52:25,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:30,350 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:52:37,068 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:52:37,068 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:52:37,069 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:52:37,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:52:38,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:39,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:39,809 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:52:39,810 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:52:41,234 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:43,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:45,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:47,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:54,971 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:52:54,972 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:52:57,109 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:52:57,110 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:52:57,110 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:52:57,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:52:58,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:52:59,243 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:00,428 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:53:01,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:03,245 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:05,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:07,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:10,103 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:53:10,103 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:53:17,152 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:53:17,152 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:53:17,153 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:53:17,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:53:18,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:19,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:21,253 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:23,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:25,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:25,297 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:53:25,297 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:53:27,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:30,506 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:53:37,189 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:53:37,189 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:53:37,190 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:53:37,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:53:38,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:39,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:40,431 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:53:40,431 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:53:41,261 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:43,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:45,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:55,562 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:53:55,562 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:53:57,226 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:53:57,227 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:53:57,227 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:53:57,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:53:58,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:53:59,269 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:00,583 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:54:01,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:03,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:05,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:10,691 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:54:10,692 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:54:17,265 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:54:17,265 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:54:17,266 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:54:17,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:54:18,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:19,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:21,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:23,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:25,281 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:25,836 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:54:25,836 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:54:30,664 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:54:37,303 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:54:37,303 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:54:37,304 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:54:38,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:54:38,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:39,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:41,001 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:54:41,002 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:54:41,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:43,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:45,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:56,133 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:54:56,133 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:54:57,340 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:54:57,340 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:54:57,340 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:54:58,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:54:58,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:54:59,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:00,744 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:55:01,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:03,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:05,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:11,264 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:55:11,264 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:55:17,378 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:55:17,379 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:55:17,379 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:55:18,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:55:18,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:19,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:21,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:23,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:25,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:26,396 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:55:26,397 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:55:30,824 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:55:37,415 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:55:37,415 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:55:37,415 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:55:38,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:55:39,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:41,313 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:41,528 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:55:41,528 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:55:43,313 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:45,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:47,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:55:56,659 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:55:56,660 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:55:57,450 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:55:57,450 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:55:57,451 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:55:58,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:55:59,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:00,905 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:56:01,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:03,322 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:05,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:07,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:11,794 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:56:11,794 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:56:17,489 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:56:17,489 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:56:17,490 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:56:18,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:56:19,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:21,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:23,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:25,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:26,927 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:56:26,927 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:56:27,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:30,987 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:56:37,533 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:56:37,533 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:56:37,534 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:56:38,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:56:39,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:41,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:42,058 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:56:42,058 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:56:43,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:45,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:47,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:56:57,193 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:56:57,193 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:56:57,570 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:56:57,571 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:56:57,571 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:56:58,350 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:56:59,350 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:01,066 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:57:01,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:03,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:05,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:12,331 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:57:12,331 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:57:17,608 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:57:17,608 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:57:17,609 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:57:18,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:57:19,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:21,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:23,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:25,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:27,461 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:57:27,461 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:57:31,146 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:57:37,648 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:57:37,648 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:57:37,649 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:57:38,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:57:39,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:41,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:42,594 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:57:42,595 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:57:43,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:45,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:57:57,686 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:57:57,686 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:57:57,687 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:57:57,743 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:57:57,744 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:57:58,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:57:59,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:01,224 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:58:01,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:03,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:05,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:12,875 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:58:12,876 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:58:17,725 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:58:17,726 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:58:17,726 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:58:18,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:58:19,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:21,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:23,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:25,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:28,006 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:58:28,007 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:58:31,299 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:58:37,762 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:58:37,762 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:58:37,763 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:58:38,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:58:39,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:41,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:43,142 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:58:43,142 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:58:43,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:45,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:58:57,800 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:58:57,800 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:58:57,801 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:58:58,280 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:58:58,280 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:58:58,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:58:59,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:01,380 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:59:01,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:03,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:05,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:13,842 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:59:13,842 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:59:17,837 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:59:17,837 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:59:17,838 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:59:18,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:59:19,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:21,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:23,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:25,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:28,977 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:59:28,977 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:59:31,458 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 10:59:37,874 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:59:37,874 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:59:37,875 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:59:38,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:59:39,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:41,423 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:43,424 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:44,288 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:59:44,288 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 10:59:45,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:57,912 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 10:59:57,912 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 10:59:57,913 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 10:59:58,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 10:59:59,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 10:59:59,453 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 10:59:59,453 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:00:01,432 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:01,537 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:00:03,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:05,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:14,592 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:00:14,593 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:00:17,950 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:00:17,951 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:00:17,951 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:00:18,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:00:19,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:21,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:23,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:25,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:29,736 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:00:29,736 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:00:31,614 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:00:37,988 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:00:37,988 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:00:37,989 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:00:38,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:00:39,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:41,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:43,450 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:44,871 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:00:44,872 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:00:45,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:00:58,027 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:00:58,028 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:00:58,028 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:00:58,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:00:59,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:00,035 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:01:00,035 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:01:01,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:01,691 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:01:03,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:05,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:15,171 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:01:15,172 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:01:18,065 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:01:18,065 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:01:18,069 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:01:18,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:01:19,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:20,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:22,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:24,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:26,469 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:30,303 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:01:30,304 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:01:31,769 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:01:38,108 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:01:38,109 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:01:38,109 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:01:38,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:01:39,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:40,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:42,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:44,477 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:45,435 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:01:45,435 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:01:46,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:01:58,146 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:01:58,146 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:01:58,147 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:01:58,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:01:59,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:00,484 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:00,565 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:02:00,566 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:02:01,846 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:02:02,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:04,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:06,487 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:15,696 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:02:15,696 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:02:18,191 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:02:18,191 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:02:18,192 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:02:18,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:02:19,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:20,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:22,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:24,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:26,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:30,829 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:02:30,830 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:02:31,925 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:02:38,232 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:02:38,232 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:02:38,233 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:02:38,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:02:39,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:40,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:42,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:44,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:45,958 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:02:45,958 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:02:46,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:02:58,272 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:02:58,272 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:02:58,273 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:02:58,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:02:59,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:00,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:01,090 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:03:01,091 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:03:02,004 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:03:02,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:04,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:06,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:16,222 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:03:16,223 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:03:18,318 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:03:18,318 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:03:18,318 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:03:18,517 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:03:19,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:20,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:22,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:24,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:26,521 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:31,355 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:03:31,355 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:03:32,084 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:03:38,355 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:03:38,355 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:03:38,356 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:03:38,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:03:39,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:40,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:42,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:44,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:46,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:03:46,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:03:46,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:03:58,393 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:03:58,393 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:03:58,393 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:03:58,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:03:59,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:00,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:01,615 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:04:01,615 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:04:02,163 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:04:02,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:04,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:06,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:16,746 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:04:16,747 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:04:18,432 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:04:18,432 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:04:18,433 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:04:18,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:04:19,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:20,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:22,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:24,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:26,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:31,886 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:04:31,886 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:04:32,240 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:04:38,471 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:04:38,471 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:04:38,471 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:04:38,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:04:39,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:40,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:42,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:44,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:46,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:04:47,018 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:04:47,019 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:04:58,512 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:04:58,512 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:04:58,512 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:04:58,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:05:00,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:02,151 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:05:02,151 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:05:02,318 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:05:02,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:04,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:06,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:17,287 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:05:17,287 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:05:18,551 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:05:18,551 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:05:18,552 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:05:18,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:05:20,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:22,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:24,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:26,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:32,399 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:05:32,426 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:05:32,427 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:05:38,592 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:05:38,592 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:05:38,593 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:05:39,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:05:40,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:42,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:44,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:46,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:05:47,557 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:05:47,557 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:05:58,631 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:05:58,631 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:05:58,632 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:05:59,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:06:00,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:02,478 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:06:02,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:02,689 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:06:02,689 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:06:04,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:06,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:17,826 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:06:17,827 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:06:18,667 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:06:18,668 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:06:18,668 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:06:19,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:06:20,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:22,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:24,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:26,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:32,556 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:06:32,961 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:06:32,961 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:06:38,705 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:06:38,706 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:06:38,706 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:06:39,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:06:40,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:42,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:44,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:46,606 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:06:48,096 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:06:48,096 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:06:58,741 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:06:58,742 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:06:58,742 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:06:59,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:07:00,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:02,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:02,633 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:07:03,234 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:07:03,235 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:07:04,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:06,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:18,366 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:07:18,366 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:07:18,778 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:07:18,778 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:07:18,779 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:07:19,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:07:20,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:22,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:24,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:26,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:32,712 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:07:33,495 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:07:33,496 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:07:38,815 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:07:38,816 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:07:38,816 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:07:39,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:07:40,630 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:42,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:44,632 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:46,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:07:48,624 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:07:48,625 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:07:58,853 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:07:58,854 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:07:58,854 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:07:59,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:08:00,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:02,640 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:02,790 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:08:03,757 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:08:03,758 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:08:04,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:06,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:18,890 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:08:18,890 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:08:18,891 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:08:18,891 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:08:18,891 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:08:19,648 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:08:20,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:22,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:24,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:26,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:32,868 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:08:34,028 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:08:34,029 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:08:38,929 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:08:38,929 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:08:38,929 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:08:39,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:08:40,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:42,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:44,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:46,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:08:49,161 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:08:49,161 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:08:58,967 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:08:58,967 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:08:58,968 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:08:59,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:09:00,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:02,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:02,945 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:09:04,291 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:09:04,292 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:09:04,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:06,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:19,003 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:09:19,004 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:09:19,004 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:09:19,433 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:09:19,433 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:09:19,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:09:20,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:22,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:24,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:26,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:33,024 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:09:34,564 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:09:34,564 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:09:39,043 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:09:39,043 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:09:39,043 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:09:39,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:09:40,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:42,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:44,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:46,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:09:49,703 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:09:49,703 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:09:59,081 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:09:59,082 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:09:59,082 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:09:59,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:10:00,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:02,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:03,101 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:10:04,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:04,833 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:10:04,833 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:10:06,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:19,119 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:10:19,119 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:10:19,120 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:10:19,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:10:19,977 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:10:19,977 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:10:20,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:22,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:24,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:26,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:33,177 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:10:35,118 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:10:35,118 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:10:39,157 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:10:39,157 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:10:39,158 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:10:39,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:10:40,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:43,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:45,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:47,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:10:50,249 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:10:50,249 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:10:59,195 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:10:59,195 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:10:59,196 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:10:59,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:11:00,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:01,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:03,254 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:11:03,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:05,382 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:11:05,383 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:11:05,719 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:07,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:09,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:11,722 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:13,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:15,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:17,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:19,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:20,517 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:11:20,518 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:11:21,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:23,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:25,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:27,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:29,730 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:31,731 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:33,333 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:11:33,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:35,650 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:11:35,651 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:11:35,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:37,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:39,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:41,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:43,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:45,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:47,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:49,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:50,788 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:11:50,789 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:11:51,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:53,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:55,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:57,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:11:59,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:03,047 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:12:03,047 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:12:03,048 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:12:03,411 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:12:03,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:12:03,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:05,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:05,920 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:12:05,920 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:12:07,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:09,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:11,750 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:21,050 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:12:21,050 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:12:23,096 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:12:23,096 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:12:23,097 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:12:23,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:12:23,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:25,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:27,756 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:29,757 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:31,758 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:33,490 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:12:36,189 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:12:36,189 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:12:43,137 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:12:43,138 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:12:43,138 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:12:43,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:12:43,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:45,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:47,765 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:49,766 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:12:51,318 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:12:51,319 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:12:51,767 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:03,178 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:13:03,179 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:13:03,179 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:13:03,566 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:13:03,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:13:03,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:05,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:06,477 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:13:06,477 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:13:07,774 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:09,775 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:11,775 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:21,612 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:13:21,612 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:13:23,217 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:13:23,217 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:13:23,218 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:13:23,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:13:23,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:25,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:27,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:29,783 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:31,784 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:33,650 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:13:36,745 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:13:36,746 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:13:43,255 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:13:43,255 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:13:43,256 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:13:43,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:13:45,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:47,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:49,791 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:51,792 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:13:51,877 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:13:51,877 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:14:03,299 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:14:03,300 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:14:03,300 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:14:03,728 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:14:03,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:14:05,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:07,010 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:14:07,010 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:14:07,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:09,800 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:11,801 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:22,140 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:14:22,140 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:14:23,338 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:14:23,339 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:14:23,339 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:14:23,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:14:25,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:27,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:29,809 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:31,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:33,808 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:14:37,271 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:14:37,271 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:14:43,377 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:14:43,378 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:14:43,378 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:14:43,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:14:45,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:47,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:49,818 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:51,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:14:52,401 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:14:52,402 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:15:03,416 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:15:03,417 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:15:03,417 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:15:03,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:15:03,884 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:15:05,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:07,531 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:15:07,531 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:15:07,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:09,827 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:11,828 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:22,680 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:15:22,680 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:15:23,455 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:15:23,455 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:15:23,455 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:15:23,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:15:25,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:28,836 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:30,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:32,838 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:33,960 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:15:37,812 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:15:37,813 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:15:43,496 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:15:43,497 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:15:43,497 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:15:43,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:15:45,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:46,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:48,845 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:50,846 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:52,847 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:15:52,941 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:15:52,941 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:16:03,533 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:16:03,533 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:16:03,534 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:16:03,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:16:04,037 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:16:04,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:05,853 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:06,853 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:08,073 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:16:08,074 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:16:08,854 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:10,855 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:12,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:23,202 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:16:23,203 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:16:23,570 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:16:23,571 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:16:23,571 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:16:23,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:16:24,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:25,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:26,863 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:28,863 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:30,864 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:32,865 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:34,115 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:16:38,336 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:16:38,337 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:16:43,607 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:16:43,608 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:16:43,608 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:16:43,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:16:44,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:45,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:46,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:48,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:50,873 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:52,874 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:16:53,472 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:16:53,473 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:17:03,644 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:17:03,645 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:17:03,645 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:17:03,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:17:04,192 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:17:05,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:06,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:08,616 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:17:08,617 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:17:08,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:10,882 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:12,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:23,685 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:17:23,685 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:17:23,686 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:17:23,813 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:17:23,813 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:17:23,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:17:25,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:26,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:28,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:30,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:32,892 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:34,264 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:17:38,946 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:17:38,946 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:17:43,722 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:17:43,723 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:17:43,723 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:17:43,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:17:44,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:45,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:46,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:48,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:50,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:52,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:17:54,081 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:17:54,082 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:18:03,761 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:18:03,762 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:18:03,762 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:18:03,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:18:04,338 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:18:04,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:05,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:06,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:08,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:09,214 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:18:09,214 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:18:10,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:12,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:23,803 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:18:23,803 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:18:23,804 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:18:23,914 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:18:24,466 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:18:24,467 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:18:24,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:25,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:26,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:28,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:30,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:32,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:34,410 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:18:39,629 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:18:39,630 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:18:43,839 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:18:43,840 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:18:43,840 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:18:43,924 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:18:44,924 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:45,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:46,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:48,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:50,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:52,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:18:54,768 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:18:54,769 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:19:03,876 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:19:03,877 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:19:03,877 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:19:03,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:19:04,484 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:19:04,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:05,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:06,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:08,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:09,902 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:19:09,902 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:19:10,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:12,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:23,915 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:19:23,916 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:19:23,916 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:19:23,941 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:19:24,941 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:25,035 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:19:25,035 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:19:25,942 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:26,942 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:28,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:30,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:32,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:34,555 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:19:40,204 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:19:40,205 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:19:43,954 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:19:43,954 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:19:43,955 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:19:44,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:19:44,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:45,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:46,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:48,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:50,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:52,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:19:55,336 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:19:55,336 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:20:03,994 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:20:03,995 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:20:03,995 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:20:04,631 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:20:04,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:20:04,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:05,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:06,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:08,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:10,497 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:20:10,498 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:20:10,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:12,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:24,033 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:20:24,034 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:20:24,034 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:20:24,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:20:24,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:25,639 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:20:25,639 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:20:25,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:26,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:28,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:30,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:32,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:34,707 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:20:40,776 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:20:40,777 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:20:44,073 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:20:44,073 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:20:44,074 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:20:44,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:20:44,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:45,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:46,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:48,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:50,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:52,982 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:20:55,906 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:20:55,906 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:21:04,111 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:21:04,111 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:21:04,111 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:21:04,784 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:21:04,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:21:04,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:05,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:06,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:08,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:10,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:11,039 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:21:11,040 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:21:12,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:24,151 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:21:24,151 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:21:24,151 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:21:24,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:21:24,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:25,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:26,175 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:21:26,175 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:21:26,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:28,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:30,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:32,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:34,859 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:21:41,312 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:21:41,313 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:21:44,186 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:21:44,186 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:21:44,186 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:21:45,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:21:45,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:47,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:49,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:51,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:53,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:21:56,444 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:21:56,445 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:22:04,224 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:22:04,225 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:22:04,225 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:22:04,934 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:22:05,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:22:05,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:07,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:09,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:11,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:11,584 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:22:11,585 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:22:13,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:24,264 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:22:24,265 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:22:24,265 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:22:25,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:22:25,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:26,717 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:22:26,718 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:22:27,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:29,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:31,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:33,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:35,007 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:22:41,855 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:22:41,855 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:22:44,305 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:22:44,305 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:22:44,305 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:22:45,030 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:22:45,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:47,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:49,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:51,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:53,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:22:56,987 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:22:56,987 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:23:04,346 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:23:04,346 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:23:04,347 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:23:05,038 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:23:05,039 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:05,084 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:23:07,039 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:09,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:11,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:12,119 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:23:12,119 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:23:13,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:24,384 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:23:24,385 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:23:24,385 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:23:25,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:23:25,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:27,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:27,285 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:23:27,286 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:23:29,048 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:31,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:33,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:35,157 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:23:42,427 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:23:42,428 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:23:44,422 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:23:44,423 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:23:44,423 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:23:45,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:23:45,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:47,057 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:49,058 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:51,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:53,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:23:57,562 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:23:57,563 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:24:04,463 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:24:04,464 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:24:04,464 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:24:05,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:24:05,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:05,231 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:24:07,074 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:09,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:11,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:12,695 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:24:12,695 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:24:13,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:24,501 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:24:24,502 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:24:24,502 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:24:25,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:24:25,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:27,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:27,828 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:24:27,828 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:24:29,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:31,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:33,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:35,309 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:24:42,961 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:24:42,962 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:24:44,538 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:24:44,538 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:24:44,539 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:24:45,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:24:46,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:47,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:48,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:50,092 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:52,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:54,094 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:24:58,093 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:24:58,094 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:25:04,575 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:25:04,575 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:25:04,575 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:25:05,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:25:05,387 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:25:06,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:07,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:08,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:10,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:12,102 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:13,226 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:25:13,226 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:25:14,103 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:24,611 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:25:24,611 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:25:24,612 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:25:25,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:25:26,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:27,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:28,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:28,357 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:25:28,357 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:25:30,110 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:32,110 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:34,111 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:35,464 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:25:43,488 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:25:43,489 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:25:44,649 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:25:44,649 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:25:44,649 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:25:45,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:25:46,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:47,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:48,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:50,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:52,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:54,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:25:58,619 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:25:58,619 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:26:04,689 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:26:04,690 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:26:04,690 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:26:05,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:26:05,542 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:26:06,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:07,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:08,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:10,126 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:12,127 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:13,752 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:26:13,752 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:26:14,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:24,726 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:26:24,726 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:26:24,727 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:26:25,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:26:26,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:27,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:28,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:28,890 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:26:28,891 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:26:30,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:32,135 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:34,136 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:35,619 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:26:44,022 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:26:44,022 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:26:44,764 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:26:44,764 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:26:44,765 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:26:45,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:26:46,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:47,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:48,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:50,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:52,144 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:54,145 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:26:59,156 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:26:59,156 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:27:04,803 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:27:04,803 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:27:04,803 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:27:05,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:27:05,696 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:27:06,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:07,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:08,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:10,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:12,152 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:14,153 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:14,288 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:27:14,288 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:27:24,843 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:27:24,843 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:27:24,843 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:27:25,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:27:26,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:27,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:28,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:29,428 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:27:29,429 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:27:30,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:32,161 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:34,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:35,775 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:27:44,579 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:27:44,579 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:27:44,879 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:27:44,879 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:27:44,880 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:27:45,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:27:46,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:47,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:48,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:50,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:52,169 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:54,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:27:59,714 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:27:59,714 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:28:04,916 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:28:04,916 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:28:04,917 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:28:05,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:28:05,852 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:28:06,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:07,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:08,176 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:10,177 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:12,178 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:14,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:14,847 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:28:14,848 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:28:24,954 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:28:24,954 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:28:24,955 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:28:25,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:28:26,184 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:27,184 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:28,185 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:29,982 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:28:29,982 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:28:30,186 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:32,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:34,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:35,944 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:28:44,992 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:28:44,993 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:28:44,993 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:28:45,117 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:28:45,117 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:28:45,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:28:46,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:47,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:48,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:50,194 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:52,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:28:54,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:00,258 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:29:00,258 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:29:05,033 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:29:05,033 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:29:05,034 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:29:05,200 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:29:06,023 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:29:06,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:07,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:08,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:10,202 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:12,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:14,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:15,393 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:29:15,393 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:29:25,072 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:29:25,072 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:29:25,073 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:29:25,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:29:26,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:27,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:28,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:30,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:30,524 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:29:30,525 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:29:32,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:34,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:36,101 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:29:45,111 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:29:45,112 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:29:45,112 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:29:45,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:29:45,709 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:29:45,709 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:29:46,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:47,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:48,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:50,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:52,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:29:54,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:00,854 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:30:00,854 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:30:05,148 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:30:05,148 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:30:05,149 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:30:05,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:30:06,179 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:30:06,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:07,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:08,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:10,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:12,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:14,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:15,985 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:30:15,986 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:30:25,185 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:30:25,186 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:30:25,186 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:30:25,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:30:26,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:27,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:28,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:30,234 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:31,117 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:30:31,118 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:30:32,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:34,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:36,256 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:30:45,225 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:30:45,225 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:30:45,226 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:30:45,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:30:46,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:46,274 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:30:46,275 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:30:47,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:48,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:50,243 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:52,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:30:54,245 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:01,415 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:31:01,416 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:31:05,267 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:31:05,268 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:31:05,268 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:31:06,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:31:06,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:06,334 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:31:07,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:08,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:10,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:12,253 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:14,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:16,549 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:31:16,549 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:31:25,305 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:31:25,305 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:31:25,306 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:31:26,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:31:26,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:27,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:28,260 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:30,261 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:31,683 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:31:31,683 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:31:32,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:34,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:36,412 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:31:45,341 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:31:45,341 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:31:45,342 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:31:46,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:31:46,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:46,863 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:31:46,863 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:31:47,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:48,269 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:50,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:52,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:31:54,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:02,012 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:32:02,012 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:32:05,377 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:32:05,377 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:32:05,378 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:32:06,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:32:06,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:06,489 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:32:07,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:08,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:10,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:12,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:14,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:17,142 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:32:17,143 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:32:25,415 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:32:25,415 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:32:25,416 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:32:26,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:32:26,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:27,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:28,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:30,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:32,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:32,300 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:32:32,301 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:32:34,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:36,567 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:32:45,454 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:32:45,454 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:32:45,455 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:32:46,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:32:46,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:47,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:47,466 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:32:47,467 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:32:48,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:50,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:52,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:32:54,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:02,602 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:33:02,602 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:33:05,491 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:33:05,491 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:33:05,492 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:33:06,302 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:33:06,645 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:33:07,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:08,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:09,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:11,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:13,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:15,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:17,733 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:33:17,733 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:33:25,530 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:33:25,530 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:33:25,531 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:33:26,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:33:27,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:28,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:29,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:31,313 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:32,893 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:33:32,894 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:33:33,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:35,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:36,723 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:33:45,567 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:33:45,567 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:33:45,567 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:33:46,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:33:47,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:48,066 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:33:48,066 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:33:48,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:49,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:51,322 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:53,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:33:55,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:03,205 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:34:03,205 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:34:05,602 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:34:05,602 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:34:05,603 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:34:06,328 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:34:06,801 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:34:07,328 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:08,329 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:09,329 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:11,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:13,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:15,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:18,363 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:34:18,363 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:34:25,639 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:34:25,640 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:34:25,640 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:34:26,336 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:34:27,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:28,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:29,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:31,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:33,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:33,494 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:34:33,494 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:34:35,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:36,879 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:34:45,675 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:34:45,675 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:34:45,676 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:34:46,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:34:47,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:48,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:48,635 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:34:48,635 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:34:49,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:51,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:53,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:34:55,349 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:03,769 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:35:03,770 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:35:05,733 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:35:05,733 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:35:05,734 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:35:06,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:35:06,956 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:35:07,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:08,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:09,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:11,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:13,356 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:15,357 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:18,909 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:35:18,909 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:35:25,771 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:35:25,772 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:35:25,772 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:35:26,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:35:27,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:28,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:29,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:31,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:33,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:34,047 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:35:34,047 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:35:35,365 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:37,033 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:35:45,811 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:35:45,811 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:35:45,812 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:35:46,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:35:47,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:48,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:49,178 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:35:49,179 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:35:49,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:51,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:53,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:35:55,373 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:04,309 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:36:04,309 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:36:05,849 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:36:05,850 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:36:05,851 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:36:06,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:36:07,113 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:36:07,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:08,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:09,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:11,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:13,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:15,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:19,471 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:36:19,471 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:36:25,889 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:36:25,890 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:36:25,890 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:36:26,385 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:36:27,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:28,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:29,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:31,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:33,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:34,604 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:36:34,604 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:36:35,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:37,189 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:36:45,926 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:36:45,927 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:36:45,927 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:36:46,393 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:36:47,393 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:48,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:49,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:49,737 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:36:49,737 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:36:51,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:36:53,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:04,869 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:37:04,870 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:37:05,964 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:37:05,964 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:37:05,965 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:37:06,401 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:37:07,266 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:37:07,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:08,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:09,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:11,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:13,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:15,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:20,002 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:37:20,002 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:37:26,003 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:37:26,004 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:37:26,004 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:37:26,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:37:27,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:28,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:29,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:31,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:33,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:35,132 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:37:35,133 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:37:35,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:37,344 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:37:46,040 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:37:46,041 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:37:46,041 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:37:46,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:37:47,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:48,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:49,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:50,280 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:37:50,280 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:37:51,420 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:53,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:37:55,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:05,411 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:38:05,412 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:38:06,077 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:38:06,077 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:38:06,078 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:38:06,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:38:07,422 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:38:07,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:08,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:09,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:11,429 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:13,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:15,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:20,543 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:38:20,544 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:38:26,118 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:38:26,118 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:38:26,118 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:38:26,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:38:27,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:28,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:29,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:31,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:33,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:35,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:35,678 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:38:35,678 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:38:37,499 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:38:46,155 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:38:46,156 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:38:46,156 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:38:46,445 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:38:47,445 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:48,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:49,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:50,818 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:38:50,818 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:38:51,447 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:53,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:38:55,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:05,949 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:39:05,950 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:39:06,195 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:39:06,196 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:39:06,196 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:39:06,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:39:07,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:07,576 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:39:08,455 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:09,455 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:11,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:13,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:15,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:21,079 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:39:21,080 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:39:26,236 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:39:26,236 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:39:26,237 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:39:26,462 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:39:27,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:28,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:29,464 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:31,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:33,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:35,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:36,212 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:39:36,212 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:39:37,654 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:39:46,276 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:39:46,276 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:39:46,277 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:39:46,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:39:47,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:48,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:49,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:51,345 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:39:51,346 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:39:51,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:53,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:39:55,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:06,319 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:40:06,319 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:40:06,320 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:40:06,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:40:06,489 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:40:06,489 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:40:07,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:07,733 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:40:08,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:09,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:11,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:13,482 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:15,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:21,627 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:40:21,628 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:40:26,357 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:40:26,358 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:40:26,358 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:40:26,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:40:26,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:27,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:29,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:31,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:33,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:35,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:36,757 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:40:36,757 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:40:37,811 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:40:46,397 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:40:46,397 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:40:46,398 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:40:46,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:40:47,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:48,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:50,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:51,888 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:40:51,889 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:40:52,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:54,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:40:56,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:06,437 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:41:06,437 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:41:06,438 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:41:06,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:41:07,046 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:41:07,047 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:41:07,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:07,889 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:41:08,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:10,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:12,508 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:14,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:16,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:22,184 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:41:22,184 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:41:26,476 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:41:26,476 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:41:26,477 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:41:26,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:41:28,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:29,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:30,516 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:32,517 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:34,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:36,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:37,316 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:41:37,317 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:41:37,970 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:41:46,512 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:41:46,512 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:41:46,513 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:41:46,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:41:48,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:49,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:50,525 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:52,452 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:41:52,452 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:41:52,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:54,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:41:56,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:06,547 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:42:06,548 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:42:06,548 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:42:07,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:42:07,604 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:42:07,605 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:42:08,048 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:42:08,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:09,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:10,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:12,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:14,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:16,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:22,767 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:42:22,767 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:42:26,584 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:42:26,584 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:42:26,585 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:42:27,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:42:28,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:29,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:30,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:32,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:34,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:36,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:37,901 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:42:37,901 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:42:38,127 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:42:46,626 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:42:46,627 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:42:46,627 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:42:47,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:42:48,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:49,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:50,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:52,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:42:53,033 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:42:53,034 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:42:54,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:06,663 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:43:06,664 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:43:06,664 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:43:07,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:43:08,167 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:43:08,168 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:43:08,297 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:43:08,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:09,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:10,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:12,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:14,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:23,328 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:43:23,329 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:43:26,699 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:43:26,699 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:43:26,700 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:43:27,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:43:28,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:29,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:30,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:32,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:34,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:38,282 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:43:38,469 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:43:38,470 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:43:46,735 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:43:46,736 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:43:46,738 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:43:47,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:43:48,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:49,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:50,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:52,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:43:53,618 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:43:53,618 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:43:54,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:06,772 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:44:06,772 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:44:06,773 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:44:07,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:44:08,360 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:44:08,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:08,783 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:44:08,783 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:44:09,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:10,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:12,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:14,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:23,929 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:44:23,934 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:44:26,815 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:44:26,816 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:44:26,818 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:44:27,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:44:27,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:28,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:30,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:32,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:34,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:38,437 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:44:39,088 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:44:39,089 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:44:46,853 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:44:46,853 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:44:46,854 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:44:47,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:44:47,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:48,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:50,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:52,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:54,221 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:44:54,221 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:44:54,606 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:44:56,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:06,890 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:45:06,891 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:45:06,893 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:45:07,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:45:07,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:08,516 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:45:08,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:09,355 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:45:09,356 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:45:10,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:12,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:14,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:16,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:24,487 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:45:24,487 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:45:26,929 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:45:26,930 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:45:26,930 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:45:27,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:45:28,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:29,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:30,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:32,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:34,624 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:36,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:38,593 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:45:39,620 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:45:39,620 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:45:46,969 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:45:46,969 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:45:46,970 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:45:47,630 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:45:48,630 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:49,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:50,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:52,632 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:54,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:45:54,751 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:45:54,751 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:45:56,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:07,007 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:46:07,007 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:46:07,008 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:46:07,638 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:46:08,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:08,670 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:46:09,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:09,907 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:46:09,907 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:46:10,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:12,640 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:14,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:16,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:25,038 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:46:25,039 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:46:27,046 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:46:27,046 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:46:27,047 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:46:27,646 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:46:28,647 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:29,647 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:30,648 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:32,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:34,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:36,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:38,748 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:46:40,168 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:46:40,168 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:46:47,085 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:46:47,085 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:46:47,086 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:46:47,655 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:46:48,655 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:49,656 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:50,656 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:52,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:54,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:46:55,302 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:46:55,302 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:47:07,123 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:47:07,124 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:47:07,124 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:47:07,663 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:47:08,664 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:08,816 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:47:09,664 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:10,437 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:47:10,437 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:47:10,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:12,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:14,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:25,571 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:47:25,571 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:47:27,161 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:47:27,161 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:47:27,162 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:47:27,672 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:47:28,672 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:29,672 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:30,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:32,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:34,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:38,887 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:47:40,703 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:47:40,703 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:47:47,197 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:47:47,198 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:47:47,198 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:47:47,680 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:47:48,681 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:49,681 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:50,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:52,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:54,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:47:55,831 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:47:55,832 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:48:07,234 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:48:07,234 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:48:07,235 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:48:07,689 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:48:08,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:08,959 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:48:09,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:10,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:10,963 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:48:10,964 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:48:12,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:14,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:26,093 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:48:26,094 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:48:27,273 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:48:27,273 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:48:27,274 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:48:27,698 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:48:28,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:29,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:30,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:32,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:34,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:39,034 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:48:41,230 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:48:41,230 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:48:47,315 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:48:47,316 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:48:47,316 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:48:47,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:48:48,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:49,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:50,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:52,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:54,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:48:56,364 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:48:56,365 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:49:07,364 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:49:07,364 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:49:07,365 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:49:07,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:49:08,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:09,103 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:49:09,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:10,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:11,692 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:49:11,692 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:49:12,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:14,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:26,829 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:49:26,829 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:49:27,409 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:49:27,409 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:49:27,410 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:49:27,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:49:27,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:28,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:30,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:32,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:34,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:39,174 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:49:41,959 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:49:41,959 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:49:47,451 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:49:47,452 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:49:47,452 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:49:47,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:49:47,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:48,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:50,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:52,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:54,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:49:57,090 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:49:57,090 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:50:07,498 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:50:07,499 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:50:07,499 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:50:07,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:50:07,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:08,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:09,241 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:50:10,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:12,220 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:50:12,221 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:50:12,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:14,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:27,360 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:50:27,361 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:50:27,538 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:50:27,538 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:50:27,539 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:50:27,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:50:29,750 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:30,750 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:31,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:33,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:35,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:39,310 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:50:42,493 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:50:42,494 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:50:47,575 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:50:47,575 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:50:47,576 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:50:47,758 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:50:49,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:50,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:51,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:53,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:55,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:50:57,622 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:50:57,622 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:51:07,613 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:51:07,613 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:51:07,614 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:51:07,767 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:51:09,377 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:51:09,767 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:10,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:11,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:12,752 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:51:12,753 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:51:13,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:15,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:27,653 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:51:27,654 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:51:27,654 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:51:27,776 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:51:27,913 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:51:27,914 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:51:28,776 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:29,776 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:31,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:33,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:35,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:39,444 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:51:43,074 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:51:43,074 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:51:47,690 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:51:47,690 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:51:47,691 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:51:47,784 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:51:48,785 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:49,785 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:51,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:53,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:55,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:57,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:51:58,205 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:51:58,205 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:52:07,725 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:52:07,725 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:52:07,726 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:52:07,793 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:52:08,794 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:09,511 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:52:09,794 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:11,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:13,355 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:52:13,356 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:52:13,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:15,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:17,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:19,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:21,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:23,800 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:25,801 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:27,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:28,491 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:52:28,491 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:52:29,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:31,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:33,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:35,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:37,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:39,581 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:52:39,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:41,809 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:43,623 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:52:43,623 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:52:43,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:45,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:47,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:49,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:51,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:53,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:55,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:57,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:52:58,779 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:52:58,779 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:52:59,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:01,818 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:03,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:05,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:07,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:09,653 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 11:53:09,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:11,581 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 11:53:11,581 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 11:53:11,582 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 11:53:11,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 11:53:13,912 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:53:13,912 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:53:18,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:20,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:22,827 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 11:53:29,044 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 11:53:29,044 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 11:53:39,727 DEBUG SenderThread:783720 [sender.py:send():179] send: stats