diff --git "a/wandb/run-20210715_142929-jw5go9rv/logs/debug-internal.log" "b/wandb/run-20210715_142929-jw5go9rv/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20210715_142929-jw5go9rv/logs/debug-internal.log" @@ -0,0 +1,1356 @@ +2021-07-15 14:29:30,489 INFO MainThread:700904 [internal.py:wandb_internal():88] W&B internal server running at pid: 700904, started at: 2021-07-15 14:29:30.489483 +2021-07-15 14:29:30,493 DEBUG SenderThread:700904 [sender.py:send():179] send: header +2021-07-15 14:29:30,493 INFO WriterThread:700904 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/run-jw5go9rv.wandb +2021-07-15 14:29:30,493 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: check_version +2021-07-15 14:29:30,493 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: check_version +2021-07-15 14:29:30,533 DEBUG SenderThread:700904 [sender.py:send():179] send: run +2021-07-15 14:29:30,727 INFO SenderThread:700904 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files +2021-07-15 14:29:30,727 INFO SenderThread:700904 [sender.py:_start_run_threads():716] run started: jw5go9rv with start time 1626359369 +2021-07-15 14:29:30,727 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:29:30,727 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: run_start +2021-07-15 14:29:30,728 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:29:31,729 INFO Thread-8 :700904 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:29:31,841 DEBUG HandlerThread:700904 [meta.py:__init__():39] meta init +2021-07-15 14:29:31,841 DEBUG HandlerThread:700904 [meta.py:__init__():53] meta init done +2021-07-15 14:29:31,841 DEBUG HandlerThread:700904 [meta.py:probe():210] probe +2021-07-15 14:29:31,842 DEBUG HandlerThread:700904 [meta.py:_setup_git():200] setup git +2021-07-15 14:29:31,872 DEBUG HandlerThread:700904 [meta.py:_setup_git():207] setup git done +2021-07-15 14:29:31,872 DEBUG HandlerThread:700904 [meta.py:_save_pip():57] save pip +2021-07-15 14:29:31,873 DEBUG HandlerThread:700904 [meta.py:_save_pip():71] save pip done +2021-07-15 14:29:31,873 DEBUG HandlerThread:700904 [meta.py:probe():252] probe done +2021-07-15 14:29:31,876 DEBUG SenderThread:700904 [sender.py:send():179] send: files +2021-07-15 14:29:31,876 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-15 14:29:31,884 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:29:31,885 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:29:32,013 DEBUG SenderThread:700904 [sender.py:send():179] send: config +2021-07-15 14:29:32,014 DEBUG SenderThread:700904 [sender.py:send():179] send: config +2021-07-15 14:29:32,014 DEBUG SenderThread:700904 [sender.py:send():179] send: config +2021-07-15 14:29:32,361 INFO Thread-11 :700904 [upload_job.py:push():137] Uploaded file /tmp/tmplntwpq4hwandb/16dfs9nn-wandb-metadata.json +2021-07-15 14:29:32,729 INFO Thread-8 :700904 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:29:32,729 INFO Thread-8 :700904 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-metadata.json +2021-07-15 14:29:32,729 INFO Thread-8 :700904 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/requirements.txt +2021-07-15 14:29:46,735 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:29:47,099 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:29:47,100 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:29:48,736 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:29:59,925 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:30:01,741 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/config.yaml +2021-07-15 14:30:02,273 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:30:02,273 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:30:17,405 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:30:17,405 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:30:30,002 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:30:32,544 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:30:32,544 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:30:47,677 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:30:47,677 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:31:00,071 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:31:02,810 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:31:02,810 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:31:17,950 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:31:17,950 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:31:30,139 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:31:33,085 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:31:33,085 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:31:48,217 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:31:48,218 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:32:00,215 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:32:03,350 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:32:03,350 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:32:18,482 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:32:18,482 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:32:30,290 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:32:33,615 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:32:33,615 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:32:48,750 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:32:48,750 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:33:00,365 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:33:03,981 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:33:03,981 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:33:19,134 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:33:19,135 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:33:30,441 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:33:34,270 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:33:34,270 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:33:49,401 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:33:49,402 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:34:00,516 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:34:04,535 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:34:04,535 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:34:19,669 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:34:19,669 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:34:30,590 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:34:34,800 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:34:34,801 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:34:49,933 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:34:49,933 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:35:00,661 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:35:05,069 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:35:05,069 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:35:17,872 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:35:20,213 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:35:20,214 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:35:30,729 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:35:35,385 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:35:35,385 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:35:50,537 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:35:50,537 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:36:00,799 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:36:05,668 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:36:05,668 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:36:20,801 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:36:20,801 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:36:30,871 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:36:35,935 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:36:35,935 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:36:51,068 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:36:51,069 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:37:00,952 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:37:06,201 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:37:06,201 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:37:21,333 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:37:21,334 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:37:25,930 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:37:31,030 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:37:36,481 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:37:36,481 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:37:51,644 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:37:51,645 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:38:01,102 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:38:06,780 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:38:06,780 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:38:21,912 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:38:21,912 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:38:31,175 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:38:37,044 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:38:37,044 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:38:52,197 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:38:52,197 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:39:01,247 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:39:07,330 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:39:07,330 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:39:22,462 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:39:22,462 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:39:31,317 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:39:37,597 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:39:37,598 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:39:44,992 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:39:46,993 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:39:48,994 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:39:50,995 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:39:52,732 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:39:52,732 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:40:01,393 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:40:03,075 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:40:03,076 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:40:03,076 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:40:04,000 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:40:04,000 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:05,000 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:07,001 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:07,877 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:40:07,877 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:40:09,002 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:11,003 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:23,010 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:40:23,010 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:40:23,141 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:40:23,141 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:40:23,141 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:40:24,009 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:40:24,009 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:25,009 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:27,010 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:29,012 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:31,013 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:31,464 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:40:38,154 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:40:38,154 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:40:43,151 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:40:43,151 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:40:43,152 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:40:44,019 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:40:44,019 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:45,019 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:47,020 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:49,021 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:51,022 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:40:53,305 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:40:53,305 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:41:01,535 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:41:03,189 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:41:03,190 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:41:03,190 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:41:04,027 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:41:05,028 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:07,029 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:08,435 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:41:08,435 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:41:09,030 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:11,031 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:23,224 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:41:23,225 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:41:23,225 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:41:23,580 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:41:23,581 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:41:24,037 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:41:25,037 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:27,038 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:29,039 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:31,040 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:31,610 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:41:38,720 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:41:38,720 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:41:43,263 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:41:43,264 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:41:43,264 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:41:44,045 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:41:45,046 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:47,047 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:49,048 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:51,049 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:41:53,851 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:41:53,852 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:42:01,685 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:42:03,299 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:42:03,300 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:42:03,300 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:42:04,055 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:42:05,055 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:07,056 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:08,991 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:42:08,991 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:42:09,057 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:11,058 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:23,336 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:42:23,337 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:42:23,337 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:42:24,063 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:42:24,147 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:42:24,147 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:42:25,064 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:27,065 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:29,066 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:31,067 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:31,760 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:42:39,281 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:42:39,282 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:42:43,374 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:42:43,374 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:42:43,375 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:42:44,072 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:42:45,073 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:47,074 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:49,075 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:51,076 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:42:54,412 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:42:54,412 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:43:01,835 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:43:03,415 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:43:03,415 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:43:03,416 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:43:04,081 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:43:05,082 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:07,082 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:09,083 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:09,548 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:43:09,548 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:43:11,084 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:23,453 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:43:23,454 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:43:23,454 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:43:24,089 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:43:24,710 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:43:24,710 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:43:25,090 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:27,090 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:29,091 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:31,092 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:31,911 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:43:39,851 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:43:39,851 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:43:43,496 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:43:43,496 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:43:43,497 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:43:44,097 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:43:45,098 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:47,099 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:49,099 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:51,100 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:43:54,983 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:43:54,983 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:44:01,990 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:44:03,535 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:44:03,535 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:44:03,536 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:44:04,106 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:44:05,107 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:07,108 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:09,108 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:10,116 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:44:10,117 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:44:11,109 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:23,574 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:44:23,575 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:44:23,575 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:44:24,115 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:44:25,115 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:25,261 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:44:25,261 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:44:27,116 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:29,117 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:31,118 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:32,067 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:44:40,397 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:44:40,398 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:44:43,616 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:44:43,616 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:44:43,616 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:44:44,123 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:44:45,124 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:47,125 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:49,126 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:51,127 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:44:55,528 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:44:55,528 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:45:02,139 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:45:03,654 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:45:03,655 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:45:03,655 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:45:04,132 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:45:05,133 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:07,134 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:09,135 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:10,671 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:45:10,671 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:45:11,135 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:23,692 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:45:23,692 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:45:23,693 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:45:24,141 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:45:25,141 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:25,830 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:45:25,830 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:45:27,142 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:29,143 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:32,144 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:32,214 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:45:40,967 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:45:40,967 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:45:43,735 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:45:43,736 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:45:43,736 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:45:44,149 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:45:45,150 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:46,150 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:48,151 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:50,152 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:52,153 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:45:56,100 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:45:56,100 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:46:02,287 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:46:03,773 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:46:03,773 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:46:03,773 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:46:04,158 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:46:05,159 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:06,159 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:08,160 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:10,161 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:11,233 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:46:11,233 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:46:12,163 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:23,812 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:46:23,813 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:46:23,813 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:46:24,168 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:46:25,168 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:26,169 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:26,367 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:46:26,367 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:46:28,170 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:30,171 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:32,172 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:32,363 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:46:41,500 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:46:41,501 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:46:43,848 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:46:43,848 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:46:43,849 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:46:44,177 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:46:45,177 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:46,178 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:48,178 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:50,179 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:52,180 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:46:56,631 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:46:56,632 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:47:02,438 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:47:03,883 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:47:03,883 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:47:03,884 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:47:04,185 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:47:05,186 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:06,186 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:08,187 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:10,188 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:11,777 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:47:11,777 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:47:12,189 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:23,922 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:47:23,923 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:47:23,923 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:47:24,195 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:47:25,195 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:26,196 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:26,909 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:47:26,910 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:47:28,197 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:30,198 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:32,198 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:32,514 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:47:42,044 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:47:42,044 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:47:43,958 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:47:43,959 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:47:43,959 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:47:44,204 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:47:45,204 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:46,205 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:48,206 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:50,207 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:52,208 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:47:57,179 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:47:57,179 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:48:02,588 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:48:03,995 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:48:03,996 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:48:03,996 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:48:04,213 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:48:05,214 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:06,214 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:08,215 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:10,216 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:12,217 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:12,310 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:48:12,311 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:48:24,032 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:48:24,032 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:48:24,033 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:48:24,223 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:48:25,223 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:26,224 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:27,440 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:48:27,440 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:48:28,225 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:30,226 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:32,227 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:32,666 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:48:42,577 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:48:42,578 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:48:44,073 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:48:44,073 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:48:44,074 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:48:44,232 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:48:45,233 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:46,233 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:48,234 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:50,235 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:52,236 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:48:57,707 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:48:57,708 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:49:02,738 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:49:04,115 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:49:04,115 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:49:04,116 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:49:04,241 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:49:05,242 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:06,242 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:08,243 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:10,244 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:12,245 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:12,839 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:49:12,839 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:49:24,152 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:49:24,152 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:49:24,153 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:49:24,250 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:49:25,250 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:26,251 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:27,967 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:49:27,968 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:49:28,252 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:30,253 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:32,254 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:32,812 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:49:43,100 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:49:43,100 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:49:44,191 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:49:44,192 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:49:44,192 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:49:44,259 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:49:45,260 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:46,260 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:48,261 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:50,262 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:52,263 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:49:58,231 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:49:58,231 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:50:02,885 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:50:04,232 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:50:04,233 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:50:04,233 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:50:04,268 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:50:05,269 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:06,269 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:08,270 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:10,271 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:12,272 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:13,361 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:50:13,361 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:50:24,273 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:50:24,273 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:50:24,274 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:50:24,276 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:50:25,277 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:26,277 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:28,278 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:28,493 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:50:28,493 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:50:30,279 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:32,280 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:32,960 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:50:43,670 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:50:43,670 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:50:44,314 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:50:44,314 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:50:44,314 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:50:45,286 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:50:45,286 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:46,286 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:48,287 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:50,288 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:52,289 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:50:58,802 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:50:58,803 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:51:03,035 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:51:04,356 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:51:04,356 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:51:04,357 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:51:05,295 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:51:05,295 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:06,295 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:08,296 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:10,297 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:12,298 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:13,934 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:51:13,934 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:51:24,396 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:51:24,396 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:51:24,396 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:51:25,303 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:51:26,304 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:28,305 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:29,064 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:51:29,064 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:51:30,306 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:32,306 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:33,111 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:51:44,195 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:51:44,196 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:51:44,432 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:51:44,432 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:51:44,433 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:51:45,312 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:51:46,312 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:48,313 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:50,314 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:52,315 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:51:59,336 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:51:59,337 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:52:03,185 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:52:04,473 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:52:04,473 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:52:04,474 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:52:05,321 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:52:06,321 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:08,322 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:10,323 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:12,324 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:14,468 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:52:14,468 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:52:24,515 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:52:24,515 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:52:24,516 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:52:25,330 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:52:26,330 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:28,331 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:29,610 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:52:29,610 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:52:30,332 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:32,333 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:33,259 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:52:44,552 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:52:44,553 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:52:44,553 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:52:44,742 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:52:44,743 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:52:45,338 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:52:46,339 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:48,340 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:50,341 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:52,342 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:52:59,911 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:52:59,912 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:53:03,325 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:53:04,589 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:53:04,589 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:53:04,589 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:53:05,347 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:53:06,347 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:15,043 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:53:15,043 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:53:26,356 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:28,356 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:30,174 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:53:30,174 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:53:30,357 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:32,358 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:33,394 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:53:34,359 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:36,360 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:38,361 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:40,362 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:42,362 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:45,337 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:53:45,337 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:53:45,363 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:47,364 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:49,365 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:51,366 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:53,367 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:55,368 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:57,369 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:53:59,370 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:00,472 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:54:00,473 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:54:01,370 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:03,371 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:03,460 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:54:05,372 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:07,373 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:09,374 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:11,375 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:13,376 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:15,377 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:15,635 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:54:15,636 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:54:17,378 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:19,379 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:21,380 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:23,381 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:25,382 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:27,383 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:29,384 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:30,771 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:54:30,772 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:54:31,385 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:33,386 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:33,529 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:54:35,387 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:37,388 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:39,389 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:41,390 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:43,264 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:54:43,264 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:54:43,264 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:54:43,390 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:54:45,391 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:45,905 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:54:45,906 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:54:47,392 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:49,393 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:54:51,394 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:01,045 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:55:01,045 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:55:03,321 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:55:03,321 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:55:03,322 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:55:03,399 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:55:03,595 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:55:04,400 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:05,400 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:08,401 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:10,402 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:12,404 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:16,537 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:55:16,537 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:55:23,362 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:55:23,363 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:55:23,363 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:55:23,409 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:55:24,409 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:26,410 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:28,411 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:30,412 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:31,676 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:55:31,676 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:55:32,413 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:33,663 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:55:43,399 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:55:43,399 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:55:43,400 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:55:43,418 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:55:44,418 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:46,420 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:46,806 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:55:46,806 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:55:48,421 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:50,422 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:55:52,423 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:01,936 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:56:01,936 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:56:03,435 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:56:03,435 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:56:03,436 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:56:03,734 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:56:04,428 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:56:04,429 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:06,429 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:08,430 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:10,431 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:12,432 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:17,077 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:56:17,078 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:56:23,470 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:56:23,470 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:56:23,470 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:56:24,437 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:56:24,438 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:26,438 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:28,439 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:30,440 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:32,274 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:56:32,274 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:56:32,441 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:33,801 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:56:43,506 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:56:43,506 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:56:43,507 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:56:44,447 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:56:44,447 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:46,447 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:47,405 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:56:47,406 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:56:48,448 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:50,449 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:56:52,450 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:02,539 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:57:02,539 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:57:03,542 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:57:03,543 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:57:03,543 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:57:03,871 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:57:04,455 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:57:05,455 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:06,456 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:08,456 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:10,457 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:12,458 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:17,673 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:57:17,673 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:57:23,581 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:57:23,581 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:57:23,581 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:57:24,463 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:57:25,464 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:26,464 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:28,465 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:30,466 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:32,466 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:32,805 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:57:32,806 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:57:33,939 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:57:43,618 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:57:43,618 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:57:43,618 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:57:44,471 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:57:45,472 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:46,472 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:47,936 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:57:47,936 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:57:48,473 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:50,474 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:57:52,475 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:03,078 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:58:03,079 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:58:03,656 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:58:03,656 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:58:03,657 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:58:04,007 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:58:04,481 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:58:05,481 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:06,482 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:08,482 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:10,483 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:12,485 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:18,210 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:58:18,210 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:58:23,693 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:58:23,693 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:58:23,694 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:58:24,490 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:58:24,490 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:25,490 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:26,491 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:28,492 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:30,493 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:32,493 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:33,341 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:58:33,342 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:58:34,080 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:58:43,732 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:58:43,732 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:58:43,733 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:58:44,499 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:58:44,499 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:45,499 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:46,500 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:48,475 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:58:48,476 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:58:48,501 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:50,502 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:58:52,503 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:03,626 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:59:03,627 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:59:03,769 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:59:03,769 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:59:03,769 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:59:04,154 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:59:04,508 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:59:04,509 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:05,509 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:06,509 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:08,510 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:10,511 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:12,512 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:18,762 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:59:18,763 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:59:23,807 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:59:23,807 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:59:23,808 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:59:24,518 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:59:24,518 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:25,518 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:26,519 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:28,519 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:30,520 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:32,522 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:33,894 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:59:33,894 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:59:34,225 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 14:59:43,863 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 14:59:43,863 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 14:59:43,864 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 14:59:44,527 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 14:59:44,527 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:45,527 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:46,528 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:48,529 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:49,023 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 14:59:49,023 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 14:59:50,530 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 14:59:52,531 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:03,903 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:00:03,903 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:00:03,904 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:00:04,171 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:00:04,171 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:00:04,304 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:00:04,537 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:00:04,537 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:05,538 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:06,538 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:08,539 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:10,540 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:12,541 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:19,335 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:00:19,336 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:00:23,940 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:00:23,940 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:00:23,941 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:00:24,546 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:00:24,547 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:25,547 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:26,547 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:28,548 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:30,549 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:32,550 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:34,369 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:00:34,561 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:00:34,562 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:00:43,978 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:00:43,979 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:00:43,979 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:00:44,555 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:00:44,555 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:45,555 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:46,556 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:48,557 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:49,693 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:00:49,693 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:00:50,558 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:00:52,559 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:04,021 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:01:04,021 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:01:04,022 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:01:04,443 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:01:04,563 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:01:04,564 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:04,836 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:01:04,836 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:01:05,564 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:06,564 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:08,565 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:10,566 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:12,567 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:19,976 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:01:19,976 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:01:24,069 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:01:24,069 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:01:24,069 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:01:24,572 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:01:24,572 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:25,572 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:26,573 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:28,574 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:30,574 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:32,575 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:34,520 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:01:35,121 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:01:35,121 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:01:44,107 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:01:44,108 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:01:44,108 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:01:44,580 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:01:44,581 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:45,581 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:46,581 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:48,582 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:50,253 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:01:50,253 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:01:51,583 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:01:53,584 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:04,151 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:02:04,152 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:02:04,152 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:02:04,589 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:02:04,593 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:02:05,396 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:02:05,397 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:02:05,589 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:07,590 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:09,591 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:11,592 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:13,593 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:20,537 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:02:20,538 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:02:24,187 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:02:24,187 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:02:24,188 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:02:24,597 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:02:25,598 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:27,599 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:29,599 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:31,600 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:33,601 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:34,667 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:02:35,674 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:02:35,674 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:02:44,224 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:02:44,224 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:02:44,225 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:02:44,606 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:02:45,606 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:47,607 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:49,608 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:50,827 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:02:50,827 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:02:51,609 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:02:53,610 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:04,260 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:03:04,261 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:03:04,261 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:03:04,615 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:03:04,743 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:03:05,615 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:05,958 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:03:05,958 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:03:07,616 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:09,617 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:11,618 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:13,619 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:21,099 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:03:21,099 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:03:24,303 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:03:24,303 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:03:24,304 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:03:24,623 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:03:25,624 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:27,624 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:29,625 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:31,626 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:33,627 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:34,816 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:03:36,230 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:03:36,230 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:03:44,341 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:03:44,341 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:03:44,341 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:03:44,632 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:03:45,632 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:47,633 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:49,634 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:51,361 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:03:51,362 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:03:51,635 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:03:53,636 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:04,380 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:04:04,380 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:04:04,380 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:04:04,640 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:04:04,892 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:04:05,641 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:06,501 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:04:06,501 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:04:07,642 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:09,643 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:11,644 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:13,645 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:21,655 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:04:21,655 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:04:24,418 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:04:24,418 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:04:24,419 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:04:24,649 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:04:25,650 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:27,651 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:29,652 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:31,653 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:33,654 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:34,966 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:04:36,789 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:04:36,789 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:04:44,452 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:04:44,453 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:04:44,453 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:04:44,658 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:04:45,659 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:47,659 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:49,660 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:51,661 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:04:51,919 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:04:51,920 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:04:53,662 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:04,489 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:05:04,489 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:05:04,489 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:05:04,667 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:05:05,040 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:05:05,668 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:07,051 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:05:07,052 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:05:07,669 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:09,670 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:11,671 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:13,672 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:22,184 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:05:22,184 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:05:24,531 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:05:24,531 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:05:24,532 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:05:24,676 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:05:25,677 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:27,678 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:29,679 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:31,680 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:33,681 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:35,115 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:05:37,320 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:05:37,320 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:05:44,569 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:05:44,569 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:05:44,570 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:05:44,685 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:05:45,686 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:47,687 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:49,688 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:51,689 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:05:52,449 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:05:52,450 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:05:53,690 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:04,605 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:06:04,606 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:06:04,606 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:06:04,694 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:06:05,189 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:06:05,695 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:07,580 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:06:07,581 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:06:07,696 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:09,697 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:11,698 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:13,699 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:22,711 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:06:22,711 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:06:24,643 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:06:24,643 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:06:24,644 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:06:24,704 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:06:25,704 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:27,705 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:29,706 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:31,707 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:33,708 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:35,263 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:06:37,858 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:06:37,858 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:06:44,683 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:06:44,683 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:06:44,684 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:06:44,712 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:06:45,712 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:47,713 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:49,714 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:51,715 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:06:53,002 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:06:53,003 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:06:53,716 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:04,722 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:07:04,722 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:07:04,723 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:07:05,339 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:07:05,721 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:07:05,721 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:06,721 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:07,722 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:08,133 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:07:08,133 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:07:09,722 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:11,723 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:13,724 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:23,264 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:07:23,265 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:07:24,763 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:07:24,764 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:07:24,764 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:07:25,730 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:07:25,730 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:26,730 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:27,731 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:29,731 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:31,732 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:33,733 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:35,415 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:07:38,396 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:07:38,396 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:07:44,841 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:07:44,842 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:07:44,842 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:07:45,738 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:07:45,739 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:46,739 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:47,739 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:49,740 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:51,741 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:07:53,540 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:07:53,541 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:07:53,742 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:04,909 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:08:04,909 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:08:04,910 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:08:05,490 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:08:05,747 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:08:05,747 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:06,747 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:07,748 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:08,672 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:08:08,672 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:08:09,749 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:11,750 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:13,750 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:15,751 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:17,752 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:19,753 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:21,754 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:23,755 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:23,835 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:08:23,835 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:08:25,756 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:28,757 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:30,758 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:32,759 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:34,760 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:35,564 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:08:36,761 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:38,762 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:38,966 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:08:38,966 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:08:40,763 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:42,764 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:44,765 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:46,766 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:48,767 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:50,768 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:52,769 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:54,096 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:08:54,096 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:08:54,770 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:56,771 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:08:58,771 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:00,772 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:02,774 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:04,775 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:05,641 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:09:06,776 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:08,777 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:09,261 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:09:09,261 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:09:10,778 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:12,779 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:14,779 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:16,780 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:18,781 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:20,782 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:22,784 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:24,394 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:09:24,394 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:09:24,724 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:09:24,724 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:09:24,725 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:09:24,785 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:09:26,786 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:28,787 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:30,788 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:32,789 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:35,715 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:09:39,554 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:09:39,555 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:09:44,775 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:09:44,775 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:09:44,776 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:09:44,794 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:09:46,795 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:48,796 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:50,797 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:52,798 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:09:54,686 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:09:54,686 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:10:04,811 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:10:04,811 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:10:04,812 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:10:05,792 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:10:05,804 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:10:06,804 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:08,805 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:09,826 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:10:09,826 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:10:10,806 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:12,807 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:24,851 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:10:24,851 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:10:24,852 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:10:24,962 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:10:24,963 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:10:25,812 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:10:26,813 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:28,814 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:30,815 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:32,816 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:35,869 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:10:40,148 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:10:40,148 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:10:44,889 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:10:44,890 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:10:44,890 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:10:45,821 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:10:46,822 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:48,823 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:50,824 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:52,825 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:10:55,281 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:10:55,281 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:11:04,928 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:11:04,928 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:11:04,929 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:11:05,831 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:11:05,942 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:11:06,831 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:08,832 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:10,475 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:11:10,476 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:11:10,833 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:12,834 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:24,968 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:11:24,968 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:11:24,969 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:11:25,633 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:11:25,634 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:11:25,839 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:11:26,839 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:28,840 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:30,841 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:32,841 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:36,017 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:11:40,770 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:11:40,771 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:11:45,012 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:11:45,012 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:11:45,013 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:11:45,847 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:11:46,848 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:48,849 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:50,849 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:52,850 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:11:55,902 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:11:55,902 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:12:05,053 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:12:05,053 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:12:05,054 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:12:05,856 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:12:06,092 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:12:06,856 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:08,857 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:10,858 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:11,032 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:12:11,033 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:12:12,859 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:25,091 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:12:25,091 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:12:25,092 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:12:25,864 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:12:26,164 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:12:26,164 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:12:26,865 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:27,865 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:29,866 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:31,867 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:33,868 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:36,168 DEBUG SenderThread:700904 [sender.py:send():179] send: stats +2021-07-15 15:12:41,331 DEBUG HandlerThread:700904 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:12:41,332 DEBUG SenderThread:700904 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:12:45,127 DEBUG SenderThread:700904 [sender.py:send():179] send: history +2021-07-15 15:12:45,127 DEBUG SenderThread:700904 [sender.py:send():179] send: summary +2021-07-15 15:12:45,127 INFO SenderThread:700904 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:12:45,873 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/wandb-summary.json +2021-07-15 15:12:46,874 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:47,874 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:49,875 INFO Thread-8 :700904 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/files/output.log +2021-07-15 15:12:50,513 WARNING MainThread:700904 [internal.py:wandb_internal():147] Internal process interrupt: 1 +2021-07-15 15:12:50,795 WARNING MainThread:700904 [internal.py:wandb_internal():147] Internal process interrupt: 2 +2021-07-15 15:12:50,795 ERROR MainThread:700904 [internal.py:wandb_internal():150] Internal process interrupted. +2021-07-15 15:12:50,948 INFO SenderThread:700904 [sender.py:finish():945] shutting down sender +2021-07-15 15:12:50,948 INFO WriterThread:700904 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_142929-jw5go9rv/run-jw5go9rv.wandb +2021-07-15 15:12:50,949 INFO SenderThread:700904 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-15 15:12:50,950 INFO HandlerThread:700904 [handler.py:finish():638] shutting down handler +2021-07-15 15:12:51,450 INFO MainThread:700904 [internal.py:handle_exit():78] Internal process exited