diff --git "a/wandb/run-20210715_171007-1mu5szt1/logs/debug-internal.log" "b/wandb/run-20210715_171007-1mu5szt1/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20210715_171007-1mu5szt1/logs/debug-internal.log" @@ -0,0 +1,815 @@ +2021-07-15 17:10:07,707 INFO MainThread:719053 [internal.py:wandb_internal():88] W&B internal server running at pid: 719053, started at: 2021-07-15 17:10:07.707137 +2021-07-15 17:10:07,709 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: check_version +2021-07-15 17:10:07,709 INFO WriterThread:719053 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/run-1mu5szt1.wandb +2021-07-15 17:10:07,710 DEBUG SenderThread:719053 [sender.py:send():179] send: header +2021-07-15 17:10:07,710 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: check_version +2021-07-15 17:10:07,748 DEBUG SenderThread:719053 [sender.py:send():179] send: run +2021-07-15 17:10:07,929 INFO SenderThread:719053 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files +2021-07-15 17:10:07,929 INFO SenderThread:719053 [sender.py:_start_run_threads():716] run started: 1mu5szt1 with start time 1626369007 +2021-07-15 17:10:07,929 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:10:07,930 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:10:07,931 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: run_start +2021-07-15 17:10:08,932 INFO Thread-8 :719053 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:10:09,048 DEBUG HandlerThread:719053 [meta.py:__init__():39] meta init +2021-07-15 17:10:09,049 DEBUG HandlerThread:719053 [meta.py:__init__():53] meta init done +2021-07-15 17:10:09,049 DEBUG HandlerThread:719053 [meta.py:probe():210] probe +2021-07-15 17:10:09,050 DEBUG HandlerThread:719053 [meta.py:_setup_git():200] setup git +2021-07-15 17:10:09,080 DEBUG HandlerThread:719053 [meta.py:_setup_git():207] setup git done +2021-07-15 17:10:09,080 DEBUG HandlerThread:719053 [meta.py:_save_pip():57] save pip +2021-07-15 17:10:09,081 DEBUG HandlerThread:719053 [meta.py:_save_pip():71] save pip done +2021-07-15 17:10:09,081 DEBUG HandlerThread:719053 [meta.py:probe():252] probe done +2021-07-15 17:10:09,084 DEBUG SenderThread:719053 [sender.py:send():179] send: files +2021-07-15 17:10:09,084 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-15 17:10:09,092 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:10:09,092 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:10:09,223 DEBUG SenderThread:719053 [sender.py:send():179] send: config +2021-07-15 17:10:09,224 DEBUG SenderThread:719053 [sender.py:send():179] send: config +2021-07-15 17:10:09,224 DEBUG SenderThread:719053 [sender.py:send():179] send: config +2021-07-15 17:10:09,549 INFO Thread-11 :719053 [upload_job.py:push():137] Uploaded file /tmp/tmp8in_nnq9wandb/29kn1aw7-wandb-metadata.json +2021-07-15 17:10:09,931 INFO Thread-8 :719053 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:10:09,932 INFO Thread-8 :719053 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-metadata.json +2021-07-15 17:10:09,932 INFO Thread-8 :719053 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/requirements.txt +2021-07-15 17:10:23,937 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:10:24,276 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:10:24,276 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:10:25,938 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:10:27,938 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:10:37,125 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:10:38,943 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/config.yaml +2021-07-15 17:10:39,471 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:10:39,472 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:10:54,603 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:10:54,603 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:11:07,196 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:11:09,759 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:11:09,759 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:11:24,892 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:11:24,892 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:11:37,265 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:11:40,025 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:11:40,025 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:11:55,155 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:11:55,156 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:12:07,334 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:12:10,289 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:12:10,289 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:12:25,436 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:12:25,436 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:12:37,404 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:12:40,568 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:12:40,568 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:12:55,701 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:12:55,702 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:13:07,473 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:13:10,836 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:13:10,836 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:13:25,968 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:13:25,968 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:13:37,544 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:13:41,099 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:13:41,099 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:13:56,233 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:13:56,233 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:14:07,616 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:14:11,368 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:14:11,368 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:14:26,502 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:14:26,502 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:14:37,688 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:14:41,639 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:14:41,639 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:14:56,770 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:14:56,771 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:15:07,766 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:15:11,905 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:15:11,906 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:15:27,035 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:15:27,036 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:15:37,844 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:15:42,168 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:15:42,168 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:15:50,062 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:15:57,310 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:15:57,310 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:16:07,956 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:16:12,465 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:16:12,465 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:16:27,596 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:16:27,596 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:16:38,038 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:16:42,727 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:16:42,728 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:16:57,864 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:16:57,864 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:17:08,119 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:17:12,998 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:17:12,999 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:17:28,128 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:17:28,128 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:17:38,201 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:17:43,267 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:17:43,268 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:17:53,116 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:17:58,414 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:17:58,414 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:18:08,274 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:18:13,567 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:18:13,567 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:18:28,696 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:18:28,696 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:18:38,348 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:18:43,830 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:18:43,831 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:18:58,961 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:18:58,962 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:19:08,421 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:19:14,095 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:19:14,095 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:19:29,226 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:19:29,227 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:19:38,495 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:19:44,359 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:19:44,359 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:19:59,501 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:19:59,502 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:20:07,175 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:08,591 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:20:09,176 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:11,176 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:13,177 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:14,638 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:20:14,639 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:20:25,346 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:20:25,347 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:20:25,347 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:20:26,182 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:20:27,183 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:29,183 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:29,779 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:20:29,779 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:20:31,184 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:33,185 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:38,668 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:20:44,912 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:20:44,912 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:20:45,388 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:20:45,388 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:20:45,389 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:20:46,190 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:20:47,190 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:49,191 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:51,192 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:20:53,193 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:00,048 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:21:00,048 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:21:05,424 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:21:05,424 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:21:05,425 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:21:06,197 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:21:07,198 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:08,744 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:21:09,199 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:11,200 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:13,200 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:15,180 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:21:15,180 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:21:25,466 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:21:25,467 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:21:25,467 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:21:26,205 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:21:27,205 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:29,206 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:30,312 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:21:30,313 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:21:31,207 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:33,208 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:38,823 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:21:45,448 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:21:45,448 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:21:45,587 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:21:45,588 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:21:45,588 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:21:46,213 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:21:47,214 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:49,215 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:51,215 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:21:53,216 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:00,622 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:22:00,623 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:22:05,559 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:22:05,559 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:22:05,560 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:22:06,222 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:22:07,222 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:08,899 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:22:09,223 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:11,224 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:13,225 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:15,754 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:22:15,754 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:22:25,599 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:22:25,599 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:22:25,600 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:22:26,230 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:22:27,231 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:29,232 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:30,892 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:22:30,892 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:22:31,233 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:33,234 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:38,977 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:22:45,638 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:22:45,639 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:22:45,639 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:22:46,030 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:22:46,030 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:22:46,239 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:22:47,239 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:49,240 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:51,241 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:22:53,242 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:01,201 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:23:01,201 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:23:05,676 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:23:05,677 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:23:05,677 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:23:06,247 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:23:07,247 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:09,055 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:23:09,248 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:11,249 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:13,250 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:16,333 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:23:16,333 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:23:25,715 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:23:25,715 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:23:25,716 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:23:26,255 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:23:27,256 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:29,257 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:31,257 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:31,466 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:23:31,466 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:23:33,258 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:39,133 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:23:45,761 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:23:45,761 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:23:45,761 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:23:46,264 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:23:46,601 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:23:46,601 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:23:47,264 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:49,265 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:51,266 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:23:53,267 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:01,740 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:24:01,741 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:24:05,798 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:24:05,798 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:24:05,798 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:24:06,272 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:24:07,272 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:09,213 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:24:09,273 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:11,274 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:13,275 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:16,872 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:24:16,872 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:24:25,842 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:24:25,842 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:24:25,843 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:24:26,280 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:24:27,281 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:29,282 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:31,283 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:32,007 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:24:32,008 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:24:33,284 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:39,289 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:24:45,881 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:24:45,881 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:24:45,882 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:24:46,288 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:24:47,162 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:24:47,162 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:24:48,289 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:50,290 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:52,291 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:24:54,292 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:02,294 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:25:02,294 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:25:05,917 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:25:05,917 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:25:05,917 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:25:06,298 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:25:08,299 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:09,365 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:25:10,300 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:12,300 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:14,301 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:17,427 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:25:17,427 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:25:25,952 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:25:25,953 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:25:25,953 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:25:26,306 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:25:28,306 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:30,307 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:32,308 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:32,561 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:25:32,561 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:25:34,309 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:39,437 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:25:45,990 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:25:45,990 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:25:45,991 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:25:46,314 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:25:47,698 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:25:47,698 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:25:48,314 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:50,315 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:52,316 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:25:54,317 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:02,836 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:26:02,837 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:26:06,028 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:26:06,029 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:26:06,030 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:26:06,322 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:26:08,323 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:09,517 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:26:10,324 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:12,324 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:14,325 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:17,976 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:26:17,977 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:26:26,064 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:26:26,064 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:26:26,065 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:26:26,330 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:26:28,331 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:30,332 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:32,332 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:33,112 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:26:33,113 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:26:34,333 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:39,594 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:26:46,102 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:26:46,103 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:26:46,103 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:26:46,338 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:26:48,244 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:26:48,244 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:26:48,339 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:50,339 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:52,340 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:26:54,341 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:03,380 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:27:03,380 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:27:06,145 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:27:06,146 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:27:06,146 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:27:06,345 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:27:08,346 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:09,671 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:27:10,346 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:12,347 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:14,348 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:18,514 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:27:18,514 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:27:26,185 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:27:26,185 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:27:26,186 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:27:26,353 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:27:28,354 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:30,354 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:32,355 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:33,646 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:27:33,647 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:27:34,356 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:39,763 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:27:46,223 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:27:46,223 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:27:46,224 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:27:46,361 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:27:48,362 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:48,779 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:27:48,780 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:27:50,363 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:52,363 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:27:54,364 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:03,915 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:28:03,915 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:28:06,261 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:28:06,262 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:28:06,262 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:28:06,369 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:28:08,370 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:09,838 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:28:10,371 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:12,371 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:14,372 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:19,048 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:28:19,049 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:28:26,302 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:28:26,302 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:28:26,303 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:28:26,378 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:28:28,378 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:30,379 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:32,380 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:34,189 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:28:34,189 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:28:34,381 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:39,915 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:28:46,339 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:28:46,340 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:28:46,340 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:28:46,385 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:28:48,386 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:49,319 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:28:49,320 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:28:50,387 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:52,388 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:28:54,389 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:04,452 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:29:04,452 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:29:06,376 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:29:06,376 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:29:06,377 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:29:06,394 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:29:08,394 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:09,993 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:29:10,395 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:12,396 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:14,397 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:19,587 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:29:19,587 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:29:26,421 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:29:26,421 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:29:26,421 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:29:27,402 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:29:28,402 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:30,403 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:32,404 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:34,405 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:34,719 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:29:34,719 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:29:40,070 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:29:46,466 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:29:46,466 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:29:46,467 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:29:47,410 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:29:48,410 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:49,852 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:29:49,853 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:29:50,411 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:52,412 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:29:54,413 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:04,983 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:30:04,984 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:30:06,504 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:30:06,504 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:30:06,505 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:30:07,418 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:30:08,419 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:10,148 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:30:10,420 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:12,421 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:14,421 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:20,165 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:30:20,166 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:30:26,542 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:30:26,543 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:30:26,543 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:30:27,427 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:30:28,427 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:30,428 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:32,429 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:34,430 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:35,296 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:30:35,297 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:30:40,226 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:30:46,584 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:30:46,584 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:30:46,584 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:30:47,435 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:30:48,435 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:50,429 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:30:50,430 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:30:50,436 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:52,437 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:30:54,437 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:05,563 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:31:05,564 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:31:06,623 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:31:06,624 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:31:06,624 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:31:07,442 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:31:08,443 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:10,304 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:31:10,443 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:12,444 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:14,445 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:20,700 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:31:20,701 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:31:26,658 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:31:26,658 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:31:26,659 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:31:27,449 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:31:28,450 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:29,450 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:31,451 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:33,452 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:35,452 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:35,832 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:31:35,832 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:31:40,382 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:31:46,697 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:31:46,698 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:31:46,698 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:31:47,457 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:31:48,457 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:49,457 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:50,964 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:31:50,964 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:31:51,458 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:53,459 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:31:55,460 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:06,095 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:32:06,096 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:32:06,732 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:32:06,733 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:32:06,733 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:32:07,464 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:32:09,465 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:10,459 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:32:11,465 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:13,466 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:15,467 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:21,227 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:32:21,227 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:32:26,779 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:32:26,779 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:32:26,780 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:32:27,471 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:32:29,472 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:31,473 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:33,474 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:35,474 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:36,358 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:32:36,358 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:32:40,536 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:32:46,811 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:32:46,812 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:32:46,812 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:32:47,478 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:32:49,479 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:51,480 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:51,488 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:32:51,488 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:32:53,481 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:32:55,481 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:06,622 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:33:06,622 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:33:06,850 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:33:06,850 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:33:06,850 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:33:07,486 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:33:09,487 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:10,614 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:33:11,487 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:13,488 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:15,489 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:21,753 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:33:21,754 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:33:26,887 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:33:26,888 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:33:26,888 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:33:27,493 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:33:29,494 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:31,495 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:33,496 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:35,497 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:36,884 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:33:36,885 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:33:40,689 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:33:46,922 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:33:46,923 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:33:46,923 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:33:47,501 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:33:49,501 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:51,502 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:52,016 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:33:52,017 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:33:53,503 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:33:55,504 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:06,961 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:34:06,961 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:34:06,962 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:34:07,157 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:34:07,157 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:34:07,508 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:34:09,509 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:10,765 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:34:11,510 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:13,511 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:15,512 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:22,299 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:34:22,300 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:34:27,005 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:34:27,005 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:34:27,006 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:34:27,516 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:34:29,517 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:31,518 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:33,519 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:35,520 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:37,435 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:34:37,435 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:34:40,843 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:34:47,057 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:34:47,057 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:34:47,057 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:34:47,526 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:34:49,526 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:51,527 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:52,567 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:34:52,567 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:34:53,528 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:34:55,529 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:07,096 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:35:07,097 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:35:07,097 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:35:07,533 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:35:07,722 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:35:07,722 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:35:09,534 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:10,920 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:35:11,535 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:13,535 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:15,536 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:22,864 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:35:22,865 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:35:27,135 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:35:27,136 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:35:27,136 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:35:27,540 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:35:29,541 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:31,541 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:33,542 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:35,543 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:38,001 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:35:38,002 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:35:40,997 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:35:47,180 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:35:47,180 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:35:47,181 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:35:47,547 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:35:49,548 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:51,549 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:53,133 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:35:53,133 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:35:53,549 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:35:55,550 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:07,226 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:36:07,227 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:36:07,227 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:36:07,554 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:36:08,286 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:36:08,287 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:36:09,555 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:11,075 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:36:11,555 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:13,556 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:15,557 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:23,428 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:36:23,428 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:36:27,264 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:36:27,264 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:36:27,264 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:36:27,562 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:36:29,563 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:31,564 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:33,565 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:35,565 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:38,696 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:36:38,696 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:36:41,154 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:36:47,305 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:36:47,305 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:36:47,306 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:36:47,569 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:36:49,570 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:51,571 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:53,572 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:36:53,828 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:36:53,828 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:36:55,573 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:07,342 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:37:07,343 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:37:07,343 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:37:07,577 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:37:08,996 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:37:08,996 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:37:09,578 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:11,232 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:37:11,579 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:13,580 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:15,581 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:24,126 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:37:24,126 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:37:27,383 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:37:27,383 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:37:27,383 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:37:27,585 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:37:29,586 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:31,587 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:33,588 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:35,589 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:39,261 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:37:39,262 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:37:41,310 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:37:47,422 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:37:47,422 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:37:47,423 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:37:47,594 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:37:49,594 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:51,595 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:53,596 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:37:54,394 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:37:54,394 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:37:55,597 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:07,468 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:38:07,468 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:38:07,469 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:38:07,602 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:38:09,530 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:38:09,530 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:38:09,603 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:11,386 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:38:11,604 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:13,605 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:15,606 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:24,689 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:38:24,689 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:38:27,506 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:38:27,507 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:38:27,507 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:38:27,611 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:38:29,612 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:31,613 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:33,614 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:35,615 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:39,823 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:38:39,823 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:38:41,464 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:38:47,548 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:38:47,548 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:38:47,548 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:38:47,619 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:38:49,620 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:51,621 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:53,622 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:38:54,951 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:38:54,952 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:38:55,623 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:07,587 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:39:07,587 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:39:07,588 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:39:07,627 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:39:09,628 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:10,117 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:39:10,118 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:39:11,543 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:39:11,629 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:13,630 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:15,631 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:25,251 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:39:25,251 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:39:27,623 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:39:27,623 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:39:27,624 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:39:27,635 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:39:29,636 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:31,637 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:33,637 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:35,638 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:40,384 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:39:40,385 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:39:41,620 DEBUG SenderThread:719053 [sender.py:send():179] send: stats +2021-07-15 17:39:47,669 DEBUG SenderThread:719053 [sender.py:send():179] send: history +2021-07-15 17:39:47,670 DEBUG SenderThread:719053 [sender.py:send():179] send: summary +2021-07-15 17:39:47,670 INFO SenderThread:719053 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 17:39:48,643 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/wandb-summary.json +2021-07-15 17:39:49,643 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:51,644 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:53,645 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:39:55,516 DEBUG HandlerThread:719053 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 17:39:55,516 DEBUG SenderThread:719053 [sender.py:send_request():193] send_request: stop_status +2021-07-15 17:39:55,646 INFO Thread-8 :719053 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/files/output.log +2021-07-15 17:40:07,245 WARNING MainThread:719053 [internal.py:wandb_internal():147] Internal process interrupt: 1 +2021-07-15 17:40:07,501 WARNING MainThread:719053 [internal.py:wandb_internal():147] Internal process interrupt: 2 +2021-07-15 17:40:07,501 ERROR MainThread:719053 [internal.py:wandb_internal():150] Internal process interrupted. +2021-07-15 17:40:07,621 INFO WriterThread:719053 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_171007-1mu5szt1/run-1mu5szt1.wandb +2021-07-15 17:40:07,830 INFO MainThread:719053 [internal.py:handle_exit():78] Internal process exited