diff --git "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" --- "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" +++ "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" @@ -18270,3 +18270,781 @@ 2021-07-16 09:29:48,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log 2021-07-16 09:29:51,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log 2021-07-16 09:29:53,518 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:29:54,712 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:29:54,713 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:29:55,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:29:57,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:29:59,521 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:01,079 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:30:01,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:03,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:05,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:07,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:09,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:09,848 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:30:09,849 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:30:11,526 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:13,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:15,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:17,529 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:19,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:21,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:22,943 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:30:22,943 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:30:22,944 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:30:23,532 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:30:24,990 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:30:24,990 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:30:25,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:27,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:29,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:31,144 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:30:31,535 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:40,124 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:30:40,124 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:30:42,997 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:30:42,997 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:30:42,998 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:30:43,540 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:30:44,540 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:45,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:47,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:49,542 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:51,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:30:55,254 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:30:55,255 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:31:01,211 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:31:03,037 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:31:03,038 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:31:03,038 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:31:03,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:31:05,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:07,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:09,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:10,384 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:31:10,385 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:31:11,552 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:23,090 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:31:23,091 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:31:23,092 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:31:23,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:31:25,540 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:31:25,541 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:31:25,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:27,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:29,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:31,281 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:31:31,561 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:40,691 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:31:40,692 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:31:43,131 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:31:43,132 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:31:43,132 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:31:43,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:31:45,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:47,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:49,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:51,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:31:55,825 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:31:55,826 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:32:01,347 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:32:03,190 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:32:03,190 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:32:03,191 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:32:03,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:32:05,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:07,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:09,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:10,955 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:32:10,956 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:32:11,578 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:23,229 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:32:23,229 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:32:23,230 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:32:23,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:32:25,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:26,102 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:32:26,102 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:32:27,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:29,586 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:31,418 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:32:32,588 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:41,232 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:32:41,233 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:32:43,522 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:32:43,523 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:32:43,523 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:32:43,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:32:44,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:45,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:46,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:48,595 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:50,596 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:52,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:32:56,365 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:32:56,365 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:33:01,484 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:33:03,590 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:33:03,591 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:33:03,591 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:33:03,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:33:04,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:05,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:06,603 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:08,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:10,605 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:11,501 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:33:11,501 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:33:12,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:23,628 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:33:23,629 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:33:23,629 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:33:24,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:33:25,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:26,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:26,632 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:33:26,632 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:33:28,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:30,614 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:31,552 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:33:32,614 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:42,178 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:33:42,178 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:33:43,665 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:33:43,666 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:33:43,666 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:33:44,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:33:45,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:46,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:48,622 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:50,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:52,624 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:33:57,316 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:33:57,317 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:34:01,616 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:34:03,703 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:34:03,704 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:34:03,704 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:34:04,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:04,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:34:05,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:06,630 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:08,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:10,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:12,460 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:34:12,460 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:34:12,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:23,742 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:34:23,743 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:34:23,744 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:34:24,638 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:24,638 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:34:25,638 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:26,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:27,602 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:34:27,603 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:34:28,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:30,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:31,681 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:34:32,642 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:42,731 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:34:42,732 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:34:43,781 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:34:43,782 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:34:43,782 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:34:44,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:44,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:34:45,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:46,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:48,649 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:50,650 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:52,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:34:57,863 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:34:57,863 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:35:01,748 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:35:03,818 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:35:03,819 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:35:03,819 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:35:04,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:35:05,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:06,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:08,658 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:10,659 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:12,660 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:12,994 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:35:12,994 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:35:23,855 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:35:23,855 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:35:23,856 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:35:24,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:35:25,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:26,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:28,127 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:35:28,127 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:35:28,667 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:30,668 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:31,814 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:35:32,669 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:43,259 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:35:43,260 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:35:43,902 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:35:43,902 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:35:43,902 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:35:44,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:44,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:35:45,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:46,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:48,676 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:50,677 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:52,678 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:35:58,397 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:35:58,397 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:36:01,880 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:36:03,941 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:36:03,941 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:36:03,941 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:36:04,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:04,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:36:05,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:06,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:08,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:10,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:12,686 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:13,529 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:36:13,530 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:36:23,983 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:36:23,984 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:36:23,984 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:36:24,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:24,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:36:25,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:26,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:28,686 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:36:28,686 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:36:28,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:30,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:31,945 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:36:32,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:43,815 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:36:43,816 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:36:44,022 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:36:44,022 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:36:44,022 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:36:44,700 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:36:45,700 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:46,700 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:48,701 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:50,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:52,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:36:58,945 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:36:58,945 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:37:02,010 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:37:04,063 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:37:04,063 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:37:04,064 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:37:04,708 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:37:05,709 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:06,709 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:08,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:10,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:12,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:14,078 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:37:14,078 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:37:24,104 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:37:24,104 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:37:24,105 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:37:24,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:24,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:37:25,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:26,718 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:28,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:29,211 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:37:29,211 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:37:30,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:32,078 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:37:32,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:44,145 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:37:44,145 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:37:44,145 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:37:44,349 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:37:44,350 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:37:44,726 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:44,726 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:37:45,726 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:46,727 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:48,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:50,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:52,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:37:59,490 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:37:59,491 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:38:02,145 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:38:04,198 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:38:04,198 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:38:04,199 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:38:04,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:04,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:38:05,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:06,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:08,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:10,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:12,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:14,622 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:38:14,622 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:38:24,242 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:38:24,242 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:38:24,243 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:38:24,743 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:38:25,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:26,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:28,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:29,754 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:38:29,755 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:38:30,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:32,212 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:38:32,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:44,279 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:38:44,280 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:38:44,280 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:38:44,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:38:44,927 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:38:44,928 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:38:45,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:46,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:48,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:50,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:38:52,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:00,070 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:39:00,071 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:39:02,276 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:39:04,334 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:39:04,335 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:39:04,335 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:39:04,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:39:05,761 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:06,761 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:08,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:10,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:12,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:15,204 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:39:15,204 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:39:24,375 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:39:24,376 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:39:24,376 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:39:24,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:24,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:39:25,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:26,770 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:28,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:30,336 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:39:30,336 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:39:30,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:32,346 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:39:32,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:44,415 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:39:44,416 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:39:44,416 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:39:44,778 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:39:45,481 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:39:45,481 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:39:45,778 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:47,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:49,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:51,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:39:53,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:00,615 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:40:00,615 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:40:02,413 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:40:04,461 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:40:04,462 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:40:04,462 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:40:04,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:40:05,788 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:07,788 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:09,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:11,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:13,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:15,744 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:40:15,745 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:40:24,525 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:40:24,525 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:40:24,526 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:40:24,796 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:40:25,796 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:27,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:29,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:30,874 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:40:30,874 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:40:31,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:32,477 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:40:33,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:44,564 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:40:44,564 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:40:44,565 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:40:44,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:40:45,805 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:46,052 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:40:46,052 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:40:47,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:49,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:51,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:40:53,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:01,188 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:41:01,188 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:41:02,548 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:41:04,602 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:41:04,603 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:41:04,603 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:41:04,813 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:41:05,814 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:07,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:09,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:11,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:13,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:16,320 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:41:16,320 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:41:24,641 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:41:24,642 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:41:24,642 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:41:24,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:41:25,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:27,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:29,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:31,460 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:41:31,460 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:41:31,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:32,616 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:41:33,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:44,679 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:41:44,680 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:41:44,680 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:41:44,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:41:45,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:46,597 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:41:46,603 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:41:47,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:49,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:51,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:41:53,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:01,757 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:42:01,757 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:42:02,684 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:42:04,717 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:42:04,718 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:42:04,718 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:42:04,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:42:05,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:07,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:09,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:11,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:13,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:16,888 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:42:16,888 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:42:24,762 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:42:24,762 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:42:24,763 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:42:24,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:42:25,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:27,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:29,852 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:31,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:32,033 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:42:32,034 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:42:32,754 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:42:33,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:44,808 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:42:44,808 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:42:44,808 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:42:44,858 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:42:45,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:47,176 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:42:47,177 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:42:47,860 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:49,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:51,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:42:53,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:02,350 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:43:02,350 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:43:02,823 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:43:04,855 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:43:04,855 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:43:04,855 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:43:04,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:43:05,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:07,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:09,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:11,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:13,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:17,483 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:43:17,484 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:43:24,912 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:43:24,912 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:43:24,913 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:43:25,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:25,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:43:27,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:29,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:31,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:32,618 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:43:32,618 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:43:32,894 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:43:33,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:44,950 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:43:44,950 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:43:44,951 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:43:45,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:45,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:43:47,750 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:43:47,751 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:43:47,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:49,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:51,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:43:53,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:02,883 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:44:02,884 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:44:03,014 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:44:04,993 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:44:04,994 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:44:04,994 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:44:05,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:05,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:44:06,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:07,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:09,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:11,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:13,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:18,016 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:44:18,016 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:44:25,034 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:44:25,035 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:44:25,035 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:44:25,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:25,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:44:26,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:27,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:29,903 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:31,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:33,040 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:44:33,152 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:44:33,152 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:44:33,905 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:45,074 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:44:45,074 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:44:45,075 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:44:45,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:45,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:44:46,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:47,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:48,283 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:44:48,283 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:44:49,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:52,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:44:54,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:03,108 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:45:03,416 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:45:03,416 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:45:05,114 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:45:05,114 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:45:05,115 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:45:05,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:45:06,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:08,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:10,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:12,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:14,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:18,548 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:45:18,548 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:45:25,153 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:45:25,154 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:45:25,154 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:45:25,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:45:26,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:28,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:30,930 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:32,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:33,176 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:45:33,679 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:45:33,679 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:45:34,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:45,194 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:45:45,194 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:45:45,195 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:45:45,937 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:45:46,938 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:48,820 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:45:48,821 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:45:48,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:50,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:52,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:45:54,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:03,249 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:46:03,952 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:46:03,953 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:46:05,231 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:46:05,232 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:46:05,232 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:46:05,946 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:46:06,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:08,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:10,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:12,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:14,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:19,087 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:46:19,088 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:46:25,289 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:46:25,289 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:46:25,289 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:46:25,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:46:26,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:28,956 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:30,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:32,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:33,323 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:46:34,216 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:46:34,217 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:46:34,959 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:45,326 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:46:45,326 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:46:45,327 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:46:45,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:46:46,965 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:48,965 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:49,347 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:46:49,348 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:46:50,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:52,967 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:46:54,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:03,398 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:47:04,479 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:47:04,479 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:47:05,387 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:47:05,387 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:47:05,388 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:47:05,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:47:06,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:08,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:10,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:12,976 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:14,977 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:19,612 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:47:19,612 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:47:25,429 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:47:25,429 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:47:25,430 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:47:25,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:47:26,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:28,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:30,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:32,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:33,472 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:47:34,743 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:47:34,743 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:47:45,468 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:47:45,469 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:47:45,469 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:47:45,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:47:46,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:48,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:49,879 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:47:49,880 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:47:50,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:52,994 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:47:54,995 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:03,549 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:48:05,009 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:48:05,010 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:48:05,505 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:48:05,506 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:48:05,506 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:48:06,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:48:07,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:09,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:11,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:13,003 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:15,004 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:20,139 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:48:20,139 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:48:25,542 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:48:25,542 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:48:25,543 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:48:26,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:48:27,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:29,010 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:31,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:33,012 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:33,622 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:48:35,013 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:35,270 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:48:35,271 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:48:45,584 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:48:45,584 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:48:45,584 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:48:46,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:48:47,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:49,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:50,399 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:48:50,400 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:48:51,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:53,021 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:48:55,022 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:03,699 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:49:05,857 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:49:05,858 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:49:05,858 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:49:06,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:49:06,047 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:49:06,047 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:49:07,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:09,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:11,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:13,030 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:15,030 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:21,185 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:49:21,185 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:49:25,895 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:49:25,895 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:49:25,896 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:49:26,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:49:27,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:29,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:31,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:33,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:33,775 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:49:35,039 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:36,376 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:49:36,377 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:49:45,932 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:49:45,932 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:49:45,932 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:49:46,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:49:47,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:49,045 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:51,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:51,510 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:49:51,511 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:49:53,047 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:49:55,048 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:03,851 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:50:05,992 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:50:05,992 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:50:05,993 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:50:06,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:50:07,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:07,158 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:50:07,158 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:50:09,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:11,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:13,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:15,056 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:22,297 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:50:22,297 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:50:26,030 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 09:50:26,030 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 09:50:26,031 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 09:50:26,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 09:50:27,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:33,926 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 09:50:34,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:36,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 09:50:37,587 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 09:50:37,588 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 09:50:38,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log