diff --git "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" --- "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" +++ "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" @@ -7896,3 +7896,3788 @@ 2021-07-16 13:37:04,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log 2021-07-16 13:37:14,469 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status 2021-07-16 13:37:14,513 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:37:25,650 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:37:29,645 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:37:29,645 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:37:44,773 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:37:44,773 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:37:55,724 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:37:59,907 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:37:59,907 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:38:15,057 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:38:15,058 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:38:16,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:18,525 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:20,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:22,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:25,801 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:38:30,199 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:38:30,199 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:38:34,349 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:38:34,350 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:38:34,350 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:38:34,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:38:36,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:38,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:40,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:42,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:45,332 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:38:45,333 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:38:54,394 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:38:54,395 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:38:54,395 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:38:54,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:38:55,877 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:38:56,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:38:58,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:00,482 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:39:00,483 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:39:00,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:02,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:14,435 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:39:14,436 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:39:14,436 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:39:14,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:39:15,614 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:39:15,614 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:39:16,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:18,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:20,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:22,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:25,948 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:39:30,753 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:39:30,753 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:39:34,474 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:39:34,475 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:39:34,475 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:39:34,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:39:36,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:38,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:40,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:42,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:45,883 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:39:45,883 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:39:54,511 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:39:54,511 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:39:54,512 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:39:54,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:39:56,015 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:39:56,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:39:58,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:00,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:01,015 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:40:01,016 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:40:02,572 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:14,547 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:40:14,547 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:40:14,548 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:40:14,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:40:16,156 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:40:16,156 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:40:16,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:18,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:20,581 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:22,581 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:26,086 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:40:31,295 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:40:31,295 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:40:34,585 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:40:34,586 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:40:34,586 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:40:34,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:40:36,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:38,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:40,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:42,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:46,428 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:40:46,428 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:40:54,623 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:40:54,623 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:40:54,624 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:40:55,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:40:56,157 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:40:56,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:40:58,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:00,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:01,562 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:41:01,563 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:41:02,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:14,662 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:41:14,662 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:41:14,663 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:41:15,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:41:16,606 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:16,697 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:41:16,697 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:41:18,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:20,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:22,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:26,226 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:41:31,852 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:41:31,852 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:41:34,700 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:41:34,701 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:41:34,702 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:41:35,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:41:36,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:38,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:40,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:42,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:46,985 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:41:46,985 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:41:54,738 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:41:54,739 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:41:54,739 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:41:55,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:41:56,294 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:41:56,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:41:58,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:00,624 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:02,118 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:42:02,118 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:42:02,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:14,776 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:42:14,776 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:42:14,777 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:42:15,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:42:16,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:17,279 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:42:17,279 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:42:18,632 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:20,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:22,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:26,363 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:42:32,412 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:42:32,413 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:42:34,813 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:42:34,813 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:42:34,814 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:42:35,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:42:36,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:38,640 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:40,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:42,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:47,544 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:42:47,544 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:42:54,853 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:42:54,853 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:42:54,854 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:42:55,647 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:42:56,435 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:42:56,647 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:42:58,648 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:00,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:02,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:02,675 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:43:02,675 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:43:14,891 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:43:14,891 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:43:14,892 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:43:15,655 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:43:16,656 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:17,808 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:43:17,808 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:43:18,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:20,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:22,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:26,507 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:43:32,944 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:43:32,945 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:43:34,928 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:43:34,928 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:43:34,929 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:43:35,664 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:43:36,664 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:38,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:40,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:42,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:48,077 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:43:48,077 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:43:54,964 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:43:54,964 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:43:54,965 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:43:55,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:43:56,583 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:43:57,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:43:59,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:01,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:03,212 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:44:03,212 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:44:03,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:15,006 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:44:15,006 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:44:15,007 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:44:15,681 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:44:17,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:18,348 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:44:18,349 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:44:19,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:21,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:23,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:26,657 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:44:33,480 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:44:33,480 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:44:35,041 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:44:35,042 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:44:35,043 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:44:35,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:44:37,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:39,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:41,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:43,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:48,619 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:44:48,619 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:44:55,078 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:44:55,079 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:44:55,079 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:44:55,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:44:56,730 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:44:57,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:44:59,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:01,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:03,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:03,750 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:45:03,751 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:45:15,116 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:45:15,116 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:45:15,117 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:45:15,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:45:17,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:18,882 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:45:18,883 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:45:19,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:21,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:23,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:26,808 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:45:34,016 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:45:34,016 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:45:35,154 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:45:35,154 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:45:35,155 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:45:35,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:45:37,719 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:39,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:41,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:43,722 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:49,149 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:45:49,149 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:45:55,197 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:45:55,198 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:45:55,198 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:45:55,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:45:56,885 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:45:57,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:45:59,730 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:01,731 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:03,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:04,281 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:46:04,282 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:46:15,236 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:46:15,237 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:46:15,237 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:46:15,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:46:17,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:19,419 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:46:19,419 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:46:19,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:21,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:23,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:26,964 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:46:34,552 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:46:34,552 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:46:35,274 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:46:35,274 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:46:35,275 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:46:35,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:46:37,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:39,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:41,750 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:43,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:49,686 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:46:49,686 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:46:55,313 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:46:55,314 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:46:55,314 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:46:55,756 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:46:57,042 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:46:57,757 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:46:59,758 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:01,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:03,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:04,823 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:47:04,823 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:47:15,351 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:47:15,351 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:47:15,352 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:47:15,766 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:47:17,767 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:19,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:19,954 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:47:19,954 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:47:21,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:23,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:27,120 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:47:35,092 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:47:35,092 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:47:35,390 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:47:35,390 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:47:35,391 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:47:35,775 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:47:37,776 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:39,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:41,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:43,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:50,223 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:47:50,223 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:47:55,426 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:47:55,426 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:47:55,427 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:47:55,785 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:47:57,197 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:47:57,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:47:59,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:01,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:03,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:05,355 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:48:05,355 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:48:15,463 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:48:15,463 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:48:15,464 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:48:15,794 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:48:17,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:19,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:20,489 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:48:20,490 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:48:21,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:23,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:27,275 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:48:35,501 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:48:35,502 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:48:35,502 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:48:35,625 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:48:35,625 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:48:35,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:48:37,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:39,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:41,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:43,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:50,756 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:48:50,757 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:48:55,561 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:48:55,562 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:48:55,562 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:48:55,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:48:57,352 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:48:57,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:48:59,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:01,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:03,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:05,898 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:49:05,899 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:49:15,600 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:49:15,601 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:49:15,601 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:49:15,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:49:17,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:19,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:21,033 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:49:21,034 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:49:21,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:23,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:27,429 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:49:35,640 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:49:35,640 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:49:35,641 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:49:35,828 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:49:36,184 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:49:36,185 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:49:37,828 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:39,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:41,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:43,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:51,323 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:49:51,324 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:49:55,680 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:49:55,680 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:49:55,681 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:49:55,836 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:49:57,506 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:49:57,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:49:59,838 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:01,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:03,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:06,455 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:50:06,455 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:50:15,721 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:50:15,722 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:50:15,722 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:50:15,845 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:50:17,846 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:19,847 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:21,589 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:50:21,590 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:50:21,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:23,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:27,584 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:50:35,760 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:50:35,760 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:50:35,761 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:50:35,854 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:50:36,726 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:50:36,726 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:50:37,855 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:39,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:41,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:43,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:51,962 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:50:51,963 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:50:55,796 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:50:55,797 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:50:55,797 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:50:55,863 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:50:57,664 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:50:57,864 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:50:59,865 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:01,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:03,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:07,099 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:51:07,099 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:51:15,839 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:51:15,839 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:51:15,840 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:51:15,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:51:17,873 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:19,874 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:21,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:22,236 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:51:22,236 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:51:23,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:27,742 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:51:35,877 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:51:35,877 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:51:35,878 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:51:35,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:51:37,382 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:51:37,382 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:51:37,882 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:39,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:41,884 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:43,885 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:52,559 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:51:52,559 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:51:55,925 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:51:55,925 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:51:55,926 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:51:56,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:51:57,818 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:51:57,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:51:59,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:01,892 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:03,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:07,698 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:52:07,698 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:52:15,965 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:52:15,966 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:52:15,966 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:52:16,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:52:17,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:19,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:21,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:22,849 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:52:22,849 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:52:23,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:27,895 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:52:36,003 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:52:36,004 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:52:36,004 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:52:36,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:52:37,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:37,989 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:52:37,989 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:52:39,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:41,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:43,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:53,160 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:52:53,161 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:52:56,042 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:52:56,043 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:52:56,043 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:52:56,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:52:57,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:52:57,973 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:52:59,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:01,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:03,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:08,293 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:53:08,293 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:53:16,081 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:53:16,082 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:53:16,082 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:53:16,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:53:17,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:19,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:21,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:23,425 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:53:23,426 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:53:23,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:28,052 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:53:36,121 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:53:36,122 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:53:36,122 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:53:36,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:53:37,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:38,556 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:53:38,556 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:53:39,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:41,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:43,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:53,692 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:53:53,692 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:53:56,169 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:53:56,169 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:53:56,170 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:53:56,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:53:57,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:53:58,130 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:53:59,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:01,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:03,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:08,825 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:54:08,825 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:54:16,210 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:54:16,211 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:54:16,211 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:54:16,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:54:17,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:19,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:21,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:23,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:23,955 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:54:23,956 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:54:28,208 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:54:36,250 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:54:36,251 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:54:36,251 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:54:36,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:54:38,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:39,086 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:54:39,087 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:54:40,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:42,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:44,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:54:54,220 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:54:54,221 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:54:56,289 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:54:56,290 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:54:56,291 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:54:56,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:54:58,286 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:54:58,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:00,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:02,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:04,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:09,353 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:55:09,353 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:55:16,329 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:55:16,329 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:55:16,330 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:55:16,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:55:18,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:20,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:22,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:24,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:55:24,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:55:24,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:28,363 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:55:36,367 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:55:36,368 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:55:36,368 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:55:36,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:55:38,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:40,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:41,223 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:55:41,224 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:55:42,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:44,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:55:56,354 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:55:56,355 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:55:56,683 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:55:56,684 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:55:56,684 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:55:56,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:55:58,441 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:55:58,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:00,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:02,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:04,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:11,484 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:56:11,484 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:56:16,722 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:56:16,722 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:56:16,723 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:56:17,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:56:19,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:21,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:23,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:25,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:26,617 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:56:26,617 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:56:28,519 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:56:36,761 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:56:36,761 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:56:36,762 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:56:37,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:56:39,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:41,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:41,751 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:56:41,751 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:56:43,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:45,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:56:56,798 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:56:56,798 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:56:56,798 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:56:56,906 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:56:56,907 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:56:57,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:56:58,601 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:56:59,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:01,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:03,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:05,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:12,045 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:57:12,046 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:57:16,836 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:57:16,836 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:57:16,837 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:57:17,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:57:19,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:21,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:23,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:25,036 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:27,178 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:57:27,179 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:57:28,679 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:57:36,876 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:57:36,877 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:57:36,877 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:57:37,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:57:39,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:41,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:42,316 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:57:42,316 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:57:43,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:45,046 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:57:56,916 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:57:56,917 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:57:56,917 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:57:57,051 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:57:57,462 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:57:57,462 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:57:58,757 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:57:59,053 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:01,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:03,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:05,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:12,600 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:58:12,601 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:58:16,954 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:58:16,955 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:58:16,955 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:58:17,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:58:19,062 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:21,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:23,065 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:25,066 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:27,066 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:27,734 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:58:27,735 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:58:28,833 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:58:29,067 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:31,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:33,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:35,070 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:37,071 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:39,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:41,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:42,867 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:58:42,867 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:58:43,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:45,074 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:47,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:49,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:51,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:54,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:56,079 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:58,021 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:58:58,022 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:58:58,079 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:58:58,913 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:59:00,080 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:02,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:04,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:06,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:08,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:10,085 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:12,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:13,159 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:59:13,159 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:59:14,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:16,087 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:18,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:20,813 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:59:20,813 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:59:20,814 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:59:21,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:59:22,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:24,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:26,092 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:28,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:28,290 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:59:28,290 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:59:28,991 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 13:59:30,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:40,859 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 13:59:40,859 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 13:59:40,860 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 13:59:41,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 13:59:42,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:43,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:43,435 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:59:43,436 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:59:44,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:46,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:48,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:50,102 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 13:59:58,575 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 13:59:58,575 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 13:59:59,069 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:00:00,897 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:00:00,897 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:00:00,898 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:00:01,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:00:02,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:03,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:04,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:06,110 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:08,111 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:10,112 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:13,704 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:00:13,705 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:00:20,933 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:00:20,933 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:00:20,934 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:00:21,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:00:22,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:24,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:26,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:28,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:28,837 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:00:28,838 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:00:29,150 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:00:30,120 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:40,971 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:00:40,971 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:00:40,972 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:00:41,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:00:42,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:43,968 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:00:43,969 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:00:44,126 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:46,127 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:48,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:50,129 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:00:59,119 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:00:59,119 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:00:59,249 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:01:01,011 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:01:01,012 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:01:01,012 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:01:01,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:01:02,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:04,135 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:06,136 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:08,136 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:10,137 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:14,250 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:01:14,251 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:01:21,049 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:01:21,049 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:01:21,049 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:01:21,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:01:22,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:24,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:26,144 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:28,145 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:29,307 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:01:29,383 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:01:29,383 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:01:30,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:41,088 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:01:41,088 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:01:41,089 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:01:41,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:01:42,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:44,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:44,518 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:01:44,518 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:01:46,152 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:48,153 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:50,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:01:59,388 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:01:59,653 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:01:59,654 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:02:01,128 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:02:01,129 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:02:01,129 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:02:01,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:02:02,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:04,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:06,161 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:08,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:10,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:14,783 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:02:14,784 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:02:21,163 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:02:21,164 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:02:21,164 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:02:21,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:02:22,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:24,169 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:26,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:28,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:29,468 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:02:29,917 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:02:29,918 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:02:30,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:41,202 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:02:41,203 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:02:41,203 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:02:42,176 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:02:42,176 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:44,177 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:45,049 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:02:45,049 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:02:46,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:48,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:50,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:02:59,546 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:03:00,181 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:03:00,181 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:03:01,241 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:03:01,242 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:03:01,242 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:03:02,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:03:02,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:04,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:06,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:08,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:10,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:15,314 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:03:15,314 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:03:21,282 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:03:21,282 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:03:21,282 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:03:22,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:03:22,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:24,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:26,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:28,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:29,626 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:03:30,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:30,447 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:03:30,447 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:03:41,319 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:03:41,320 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:03:41,320 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:03:42,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:03:42,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:44,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:45,580 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:03:45,580 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:03:46,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:48,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:50,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:03:59,705 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:04:00,718 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:04:00,718 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:04:01,358 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:04:01,359 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:04:01,359 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:04:02,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:04:02,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:04,214 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:06,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:08,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:10,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:15,852 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:04:15,852 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:04:21,398 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:04:21,398 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:04:21,399 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:04:22,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:04:22,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:24,223 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:26,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:28,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:29,784 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:04:30,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:31,059 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:04:31,059 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:04:41,439 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:04:41,439 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:04:41,440 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:04:42,230 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:04:42,230 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:44,231 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:46,208 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:04:46,208 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:04:46,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:48,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:50,234 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:04:59,862 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:05:01,346 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:05:01,347 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:05:01,477 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:05:01,478 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:05:01,478 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:05:02,239 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:05:02,239 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:04,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:06,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:08,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:10,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:16,510 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:05:16,511 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:05:21,511 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:05:21,511 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:05:21,512 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:05:22,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:05:22,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:24,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:26,249 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:28,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:29,942 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:05:30,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:31,643 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:05:31,644 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:05:41,548 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:05:41,549 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:05:41,549 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:05:42,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:05:42,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:44,257 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:46,257 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:46,779 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:05:46,779 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:05:48,258 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:05:50,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:00,022 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:06:01,585 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:06:01,585 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:06:01,586 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:06:01,947 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:06:01,947 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:06:02,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:06:02,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:04,266 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:07,267 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:09,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:11,269 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:17,089 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:06:17,089 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:06:21,621 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:06:21,621 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:06:21,621 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:06:22,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:06:23,275 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:25,276 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:27,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:29,278 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:30,104 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:06:31,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:32,221 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:06:32,221 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:06:41,658 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:06:41,658 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:06:41,658 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:06:42,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:06:43,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:45,284 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:47,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:47,361 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:06:47,361 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:06:49,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:06:51,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:00,181 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:07:01,693 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:07:01,693 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:07:01,694 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:07:02,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:07:02,538 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:07:02,538 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:07:03,292 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:05,292 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:07,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:09,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:11,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:17,674 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:07:17,675 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:07:21,728 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:07:21,728 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:07:21,729 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:07:22,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:07:23,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:25,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:27,302 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:29,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:30,260 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:07:31,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:32,808 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:07:32,808 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:07:41,762 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:07:41,763 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:07:41,763 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:07:42,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:07:43,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:45,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:47,310 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:47,944 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:07:47,945 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:07:49,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:07:51,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:00,337 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:08:01,805 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:08:01,805 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:08:01,806 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:08:02,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:08:03,081 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:08:03,081 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:08:03,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:05,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:07,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:09,319 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:11,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:18,252 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:08:18,253 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:08:21,841 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:08:21,842 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:08:21,842 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:08:22,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:08:23,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:25,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:27,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:29,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:30,418 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:08:31,328 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:33,385 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:08:33,385 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:08:41,878 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:08:41,879 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:08:41,879 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:08:42,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:08:43,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:45,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:47,335 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:48,522 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:08:48,522 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:08:49,336 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:08:51,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:00,497 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:09:01,916 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:09:01,917 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:09:01,917 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:09:02,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:09:03,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:03,653 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:09:03,654 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:09:05,343 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:07,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:09,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:11,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:18,814 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:09:18,815 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:09:21,955 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:09:21,955 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:09:21,956 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:09:22,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:09:23,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:25,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:27,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:29,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:30,575 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:09:31,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:33,944 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:09:33,945 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:09:41,996 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:09:41,996 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:09:41,997 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:09:42,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:09:43,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:45,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:47,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:49,073 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:09:49,074 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:09:49,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:09:51,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:00,652 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:10:02,035 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:10:02,036 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:10:02,036 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:10:02,368 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:10:03,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:04,213 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:10:04,214 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:10:05,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:07,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:09,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:11,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:19,351 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:10:19,351 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:10:22,074 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:10:22,075 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:10:22,076 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:10:22,376 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:10:23,377 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:25,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:27,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:29,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:30,730 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:10:31,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:34,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:10:34,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:10:42,115 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:10:42,116 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:10:42,116 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:10:42,385 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:10:43,385 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:45,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:47,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:49,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:10:49,616 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:10:49,617 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:10:51,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:00,808 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:11:02,153 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:11:02,154 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:11:02,154 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:11:02,393 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:11:03,393 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:04,750 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:11:04,750 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:11:05,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:07,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:09,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:11,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:19,889 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:11:19,889 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:11:22,198 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:11:22,198 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:11:22,199 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:11:22,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:11:23,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:25,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:27,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:29,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:30,888 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:11:31,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:35,024 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:11:35,025 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:11:42,473 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:11:42,473 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:11:42,473 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:11:43,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:11:43,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:45,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:47,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:49,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:11:50,153 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:11:50,154 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:11:51,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:00,963 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:12:02,512 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:12:02,512 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:12:02,513 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:12:03,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:12:03,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:05,291 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:12:05,291 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:12:05,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:07,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:09,420 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:11,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:20,432 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:12:20,433 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:12:22,549 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:12:22,550 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:12:22,550 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:12:23,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:12:23,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:25,426 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:27,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:29,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:31,041 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:12:31,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:35,563 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:12:35,563 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:12:42,590 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:12:42,591 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:12:42,591 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:12:43,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:12:43,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:45,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:47,435 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:49,435 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:12:50,695 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:12:50,695 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:12:51,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:01,118 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:13:02,629 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:13:02,630 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:13:02,630 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:13:03,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:13:03,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:05,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:05,827 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:13:05,827 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:13:07,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:09,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:11,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:20,960 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:13:20,960 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:13:22,667 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:13:22,667 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:13:22,667 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:13:23,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:13:23,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:25,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:27,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:29,450 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:31,194 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:13:31,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:36,094 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:13:36,094 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:13:42,705 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:13:42,705 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:13:42,706 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:13:43,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:13:43,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:45,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:47,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:49,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:13:51,226 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:13:51,226 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:13:51,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:01,262 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:14:02,744 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:14:02,744 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:14:02,744 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:14:03,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:14:03,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:04,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:06,360 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:14:06,360 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:14:06,464 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:08,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:10,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:12,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:21,496 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:14:21,497 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:14:22,780 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:14:22,781 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:14:22,781 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:14:23,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:14:23,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:24,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:26,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:28,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:30,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:31,331 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:14:32,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:36,627 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:14:36,628 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:14:42,817 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:14:42,817 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:14:42,818 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:14:43,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:14:43,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:44,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:46,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:48,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:50,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:14:51,760 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:14:51,760 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:14:52,482 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:01,400 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:15:02,857 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:15:02,857 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:15:02,858 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:15:03,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:15:03,487 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:04,487 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:06,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:06,892 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:15:06,893 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:15:08,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:10,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:12,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:22,026 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:15:22,027 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:15:22,899 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:15:22,900 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:15:22,900 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:15:23,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:15:23,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:24,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:26,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:28,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:30,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:31,469 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:15:32,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:37,158 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:15:37,158 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:15:42,939 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:15:42,940 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:15:42,940 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:15:43,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:15:43,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:44,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:46,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:48,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:50,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:15:52,289 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:15:52,289 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:15:52,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:01,537 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:16:02,980 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:16:02,981 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:16:02,981 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:16:03,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:16:03,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:04,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:06,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:07,422 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:16:07,422 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:16:08,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:10,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:12,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:22,555 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:16:22,555 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:16:23,018 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:16:23,018 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:16:23,019 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:16:23,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:16:23,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:24,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:26,521 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:28,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:30,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:31,607 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:16:32,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:37,686 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:16:37,686 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:16:43,060 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:16:43,060 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:16:43,061 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:16:43,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:16:44,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:46,530 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:48,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:50,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:52,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:16:52,817 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:16:52,817 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:17:01,676 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:17:03,100 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:17:03,100 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:17:03,100 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:17:03,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:17:04,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:06,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:07,953 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:17:07,954 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:17:08,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:10,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:12,540 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:23,089 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:17:23,090 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:17:23,222 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:17:23,223 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:17:23,223 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:17:23,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:17:24,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:26,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:28,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:30,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:31,746 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:17:32,548 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:38,246 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:17:38,246 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:17:43,179 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:17:43,179 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:17:43,180 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:17:43,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:17:44,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:46,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:48,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:50,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:52,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:17:53,379 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:17:53,379 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:18:01,814 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:18:03,219 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:18:03,219 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:18:03,220 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:18:03,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:18:04,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:06,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:08,509 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:18:08,510 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:18:08,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:10,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:12,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:23,257 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:18:23,257 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:18:23,258 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:18:23,567 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:18:23,673 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:18:23,674 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:18:24,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:26,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:28,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:30,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:31,886 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:18:32,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:38,811 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:18:38,811 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:18:43,293 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:18:43,294 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:18:43,294 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:18:43,575 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:18:44,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:46,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:48,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:50,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:52,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:18:53,945 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:18:53,945 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:19:01,958 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:19:03,329 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:19:03,329 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:19:03,330 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:19:03,583 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:19:04,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:06,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:08,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:09,081 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:19:09,081 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:19:10,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:12,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:23,365 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:19:23,366 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:19:23,366 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:19:23,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:19:24,559 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:19:24,559 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:19:24,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:26,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:28,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:30,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:32,036 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:19:32,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:39,720 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:19:39,721 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:19:43,402 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:19:43,402 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:19:43,403 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:19:43,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:19:44,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:46,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:48,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:50,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:52,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:19:54,857 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:19:54,857 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:20:02,111 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:20:03,438 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:20:03,439 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:20:03,439 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:20:03,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:20:04,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:06,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:08,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:10,162 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:20:10,162 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:20:10,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:12,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:23,474 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:20:23,474 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:20:23,475 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:20:23,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:20:24,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:25,328 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:20:25,329 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:20:26,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:28,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:30,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:32,189 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:20:32,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:40,467 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:20:40,467 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:20:43,511 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:20:43,511 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:20:43,512 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:20:43,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:20:44,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:46,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:48,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:50,628 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:52,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:20:55,598 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:20:55,598 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:21:02,264 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:21:03,547 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:21:03,548 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:21:03,548 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:21:03,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:21:04,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:06,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:08,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:10,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:10,730 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:21:10,731 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:21:12,637 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:23,586 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:21:23,586 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:21:23,586 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:21:23,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:21:24,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:25,870 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:21:25,870 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:21:26,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:28,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:30,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:32,344 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:21:32,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:41,028 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:21:41,028 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:21:43,624 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:21:43,624 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:21:43,625 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:21:43,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:21:44,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:46,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:48,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:50,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:52,654 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:21:56,161 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:21:56,162 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:22:02,422 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:22:03,662 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:22:03,662 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:22:03,663 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:22:04,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:22:04,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:06,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:08,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:10,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:11,292 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:22:11,293 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:22:12,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:23,700 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:22:23,701 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:22:23,701 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:22:24,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:22:24,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:26,424 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:22:26,424 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:22:26,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:28,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:30,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:32,500 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:22:32,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:41,557 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:22:41,557 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:22:43,739 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:22:43,740 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:22:43,740 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:22:44,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:22:44,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:46,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:48,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:50,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:52,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:22:56,688 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:22:56,688 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:23:02,577 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:23:03,777 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:23:03,777 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:23:03,778 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:23:04,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:23:04,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:06,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:08,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:10,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:11,820 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:23:11,821 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:23:12,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:23,816 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:23:23,817 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:23:23,817 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:23:24,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:23:24,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:26,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:26,953 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:23:26,953 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:23:28,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:30,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:32,655 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:23:32,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:42,116 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:23:42,116 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:23:43,856 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:23:43,856 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:23:43,856 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:23:44,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:23:44,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:46,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:48,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:50,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:52,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:23:57,249 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:23:57,249 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:24:02,733 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:24:03,892 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:24:03,893 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:24:03,893 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:24:04,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:24:04,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:06,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:08,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:10,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:12,387 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:24:12,387 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:24:12,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:23,932 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:24:23,933 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:24:23,933 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:24:24,714 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:24:25,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:27,526 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:24:27,527 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:24:27,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:29,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:31,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:32,811 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:24:33,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:42,662 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:24:42,662 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:24:43,971 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:24:43,972 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:24:43,972 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:24:44,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:24:45,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:47,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:49,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:51,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:53,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:24:57,796 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:24:57,796 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:25:02,889 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:25:04,010 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:25:04,010 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:25:04,011 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:25:04,731 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:25:05,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:07,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:09,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:11,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:12,929 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:25:12,929 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:25:13,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:24,046 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:25:24,047 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:25:24,047 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:25:24,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:25:25,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:27,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:28,060 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:25:28,060 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:25:29,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:31,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:32,967 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:25:33,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:43,194 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:25:43,195 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:25:44,084 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:25:44,084 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:25:44,085 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:25:44,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:25:45,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:47,750 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:49,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:51,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:25:58,328 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:25:58,328 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:26:03,044 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:26:04,123 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:26:04,124 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:26:04,124 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:26:04,757 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:26:05,758 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:07,758 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:09,759 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:11,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:13,461 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:26:13,461 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:26:13,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:24,161 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:26:24,162 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:26:24,162 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:26:24,766 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:26:25,767 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:27,767 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:28,597 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:26:28,597 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:26:29,768 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:31,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:33,120 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:26:33,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:43,730 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:26:43,730 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:26:44,197 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:26:44,197 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:26:44,198 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:26:44,775 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:26:45,775 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:47,776 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:49,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:51,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:53,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:26:58,863 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:26:58,864 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:27:03,196 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:27:04,237 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:27:04,237 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:27:04,237 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:27:04,784 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:27:05,784 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:07,785 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:09,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:11,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:13,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:13,996 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:27:13,997 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:27:24,275 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:27:24,275 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:27:24,276 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:27:24,793 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:27:25,794 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:27,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:29,130 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:27:29,130 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:27:29,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:31,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:33,274 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:27:33,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:44,264 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:27:44,264 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:27:44,413 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:27:44,413 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:27:44,414 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:27:44,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:27:45,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:47,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:49,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:51,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:53,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:27:59,604 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:27:59,605 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:28:03,351 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:28:04,582 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:28:04,582 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:28:04,583 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:28:04,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:28:05,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:07,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:09,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:11,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:13,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:14,737 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:28:14,738 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:28:24,616 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:28:24,617 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:28:24,617 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:28:24,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:28:25,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:27,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:29,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:29,988 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:28:29,988 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:28:31,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:33,428 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:28:33,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:44,653 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:28:44,653 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:28:44,654 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:28:44,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:28:45,132 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:28:45,132 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:28:45,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:47,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:49,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:51,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:28:53,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:00,268 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:29:00,269 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:29:03,504 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:29:04,688 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:29:04,688 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:29:04,689 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:29:04,838 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:29:05,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:07,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:09,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:11,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:13,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:15,405 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:29:15,406 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:29:24,723 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:29:24,724 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:29:24,724 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:29:24,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:29:25,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:27,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:29,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:30,538 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:29:30,539 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:29:31,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:33,582 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:29:33,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:44,761 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:29:44,761 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:29:44,761 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:29:44,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:29:45,674 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:29:45,674 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:29:45,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:47,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:49,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:51,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:29:53,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:00,812 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:30:00,812 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:30:03,661 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:30:04,798 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:30:04,799 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:30:04,799 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:30:04,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:30:05,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:07,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:09,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:11,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:13,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:15,944 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:30:15,944 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:30:24,838 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:30:24,839 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:30:24,839 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:30:24,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:30:25,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:27,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:29,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:31,078 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:30:31,078 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:30:31,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:33,739 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:30:33,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:44,878 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:30:44,879 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:30:44,879 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:30:44,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:30:45,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:46,227 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:30:46,227 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:30:47,884 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:49,885 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:51,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:30:53,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:01,366 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:31:01,366 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:31:03,816 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:31:04,917 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:31:04,917 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:31:04,918 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:31:05,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:31:05,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:07,892 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:09,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:11,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:13,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:16,499 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:31:16,500 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:31:24,958 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:31:24,958 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:31:24,958 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:31:25,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:31:25,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:27,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:29,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:31,632 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:31:31,632 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:31:31,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:33,894 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:31:33,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:44,996 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:31:44,997 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:31:44,997 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:31:45,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:31:45,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:46,768 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:31:46,769 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:31:47,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:49,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:51,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:31:53,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:01,910 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:32:01,910 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:32:03,972 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:32:05,036 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:32:05,037 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:32:05,037 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:32:05,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:32:05,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:07,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:09,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:11,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:13,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:17,042 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:32:17,042 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:32:25,078 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:32:25,078 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:32:25,078 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:32:25,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:32:25,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:27,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:29,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:31,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:32,174 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:32:32,174 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:32:33,930 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:34,048 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:32:45,119 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:32:45,119 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:32:45,119 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:32:45,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:32:45,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:47,309 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:32:47,309 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:32:47,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:49,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:51,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:32:53,939 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:02,470 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:33:02,470 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:33:04,127 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:33:05,159 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:33:05,159 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:33:05,159 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:33:05,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:33:05,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:07,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:09,947 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:11,948 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:13,949 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:17,613 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:33:17,613 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:33:25,201 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:33:25,202 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:33:25,202 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:33:25,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:33:25,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:27,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:29,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:31,956 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:32,744 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:33:32,745 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:33:33,957 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:34,205 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:33:45,265 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:33:45,265 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:33:45,265 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:33:45,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:33:46,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:48,031 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:33:48,032 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:33:48,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:50,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:52,965 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:33:54,966 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:03,166 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:34:03,166 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:34:04,283 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:34:05,304 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:34:05,304 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:34:05,305 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:34:05,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:34:06,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:08,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:10,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:12,974 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:14,975 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:18,377 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:34:18,377 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:34:25,343 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:34:25,343 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:34:25,344 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:34:25,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:34:26,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:28,982 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:30,983 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:32,984 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:33,515 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:34:33,516 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:34:34,360 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:34:34,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:45,380 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:34:45,381 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:34:45,381 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:34:45,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:34:46,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:48,680 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:34:48,680 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:34:48,991 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:50,992 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:52,993 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:34:54,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:03,811 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:35:03,812 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:35:04,438 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:35:05,423 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:35:05,423 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:35:05,424 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:35:05,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:35:06,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:09,000 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:11,001 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:13,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:15,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:18,942 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:35:18,942 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:35:25,462 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:35:25,463 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:35:25,463 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:35:26,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:35:27,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:29,009 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:31,010 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:33,011 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:34,073 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:35:34,073 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:35:34,515 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:35:35,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:45,499 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:35:45,499 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:35:45,500 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:35:46,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:35:47,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:49,018 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:49,219 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:35:49,220 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:35:51,019 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:53,020 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:35:55,021 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:04,354 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:36:04,354 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:36:04,592 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:36:05,535 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:36:05,536 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:36:05,536 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:36:06,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:36:07,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:09,027 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:11,028 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:13,029 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:15,030 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:19,486 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:36:19,486 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:36:25,576 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:36:25,577 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:36:25,577 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:36:26,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:36:27,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:29,036 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:31,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:33,038 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:34,618 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:36:34,618 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:36:34,748 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:36:35,039 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:45,617 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:36:45,617 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:36:45,618 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:36:46,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:36:47,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:49,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:49,749 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:36:49,750 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:36:51,046 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:53,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:36:55,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:04,743 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:37:05,125 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:37:05,125 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:37:05,657 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:37:05,657 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:37:05,660 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:37:06,052 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:37:07,053 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:09,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:11,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:13,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:15,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:20,257 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:37:20,258 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:37:25,698 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:37:25,699 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:37:25,699 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:37:26,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:37:27,062 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:29,062 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:31,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:33,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:34,820 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:37:35,065 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:35,391 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:37:35,391 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:37:45,734 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:37:45,735 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:37:45,737 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:37:46,070 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:37:47,070 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:49,071 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:50,522 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:37:50,522 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:37:51,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:53,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:37:55,074 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:04,889 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:38:05,653 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:38:05,653 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:38:05,784 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:38:05,784 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:38:05,785 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:38:06,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:38:07,079 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:09,080 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:11,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:13,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:15,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:20,812 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:38:20,813 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:38:25,811 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:38:25,811 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:38:25,813 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:38:26,087 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:38:27,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:29,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:31,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:33,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:34,956 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:38:35,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:35,943 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:38:35,943 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:38:45,846 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:38:45,846 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:38:45,847 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:38:46,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:38:47,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:49,097 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:51,076 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:38:51,076 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:38:51,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:53,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:38:55,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:05,024 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:39:05,912 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:39:05,912 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:39:05,912 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:39:06,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:39:06,206 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:39:06,207 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:39:07,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:09,105 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:11,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:13,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:15,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:21,344 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:39:21,344 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:39:25,948 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:39:25,948 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:39:25,949 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:39:26,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:39:27,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:29,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:31,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:33,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:35,094 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:39:35,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:36,474 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:39:36,475 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:39:37,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:39,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:41,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:43,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:45,120 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:47,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:49,122 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:51,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:51,610 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:39:51,610 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:39:53,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:55,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:57,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:39:59,126 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:01,127 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:03,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:05,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:05,161 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:40:06,762 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:40:06,762 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:40:08,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:10,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:12,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:14,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:16,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:18,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:20,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:21,898 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:40:21,898 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:40:22,136 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:24,136 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:26,137 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:28,138 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:29,932 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:40:29,932 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:40:29,933 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:40:30,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:40:32,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:34,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:35,229 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:40:36,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:37,035 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:40:37,035 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:40:38,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:49,979 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:40:49,979 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:40:49,979 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:40:50,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:40:51,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:52,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:52,350 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:40:52,351 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:40:54,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:56,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:40:58,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:05,297 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:41:07,494 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:41:07,495 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:41:10,016 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:41:10,016 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:41:10,017 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:41:10,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:41:11,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:12,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:14,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:16,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:18,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:22,628 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:41:22,629 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:41:30,057 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:41:30,058 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:41:30,058 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:41:30,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:41:31,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:32,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:34,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:35,366 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:41:36,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:37,769 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:41:37,770 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:41:38,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:50,095 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:41:50,095 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:41:50,096 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:41:50,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:41:51,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:52,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:52,900 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:41:52,901 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:41:54,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:56,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:41:58,176 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:05,436 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:42:08,035 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:42:08,035 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:42:10,132 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:42:10,132 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:42:10,132 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:42:10,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:42:11,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:12,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:14,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:16,185 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:18,186 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:23,170 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:42:23,170 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:42:30,170 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:42:30,171 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:42:30,171 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:42:30,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:42:31,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:32,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:34,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:35,505 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:42:36,194 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:38,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:38,303 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:42:38,303 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:42:50,208 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:42:50,209 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:42:50,209 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:42:51,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:42:52,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:53,442 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:42:53,442 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:42:54,202 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:56,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:42:58,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:05,573 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:43:08,578 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:43:08,578 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:43:10,246 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:43:10,246 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:43:10,247 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:43:11,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:43:12,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:14,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:16,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:18,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:23,710 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:43:23,710 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:43:30,286 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:43:30,286 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:43:30,287 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:43:31,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:43:32,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:34,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:35,645 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:43:36,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:38,221 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:38,840 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:43:38,840 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:43:50,550 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:43:50,551 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:43:50,551 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:43:51,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:43:52,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:53,971 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:43:53,971 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:43:54,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:56,229 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:43:58,230 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:05,713 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:44:09,113 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:44:09,113 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:44:10,590 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:44:10,591 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:44:10,591 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:44:11,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:44:12,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:14,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:16,238 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:18,239 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:20,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:24,244 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:44:24,244 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:44:30,627 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:44:30,628 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:44:30,628 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:44:31,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:44:32,245 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:34,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:35,781 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:44:36,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:38,247 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:39,380 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:44:39,380 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:44:40,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:50,662 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:44:50,663 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:44:50,663 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:44:51,253 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:44:52,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:54,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:54,510 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:44:54,510 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:44:56,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:44:58,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:00,257 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:05,849 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:45:09,640 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:45:09,640 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:45:10,701 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:45:10,701 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:45:10,702 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:45:11,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:45:12,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:14,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:16,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:18,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:20,266 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:24,774 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:45:24,774 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:45:30,742 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:45:30,742 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:45:30,743 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:45:31,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:45:32,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:34,273 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:35,920 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:45:36,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:38,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:39,915 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:45:39,915 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:45:40,275 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:50,800 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:45:50,801 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:45:50,801 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:45:51,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:45:52,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:54,281 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:55,047 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:45:55,047 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:45:56,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:45:58,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:05,995 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:46:10,180 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:46:10,181 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:46:10,850 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:46:10,850 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:46:10,851 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:46:11,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:46:12,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:14,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:16,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:18,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:25,315 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:46:25,315 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:46:30,889 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:46:30,889 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:46:30,889 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:46:31,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:46:32,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:33,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:35,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:36,075 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:46:37,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:39,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:40,447 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:46:40,447 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:46:50,927 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:46:50,927 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:46:50,927 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:46:51,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:46:52,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:53,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:55,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:55,581 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:46:55,581 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:46:57,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:46:59,307 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:06,149 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:47:10,713 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:47:10,713 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:47:10,967 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:47:10,967 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:47:10,968 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:47:11,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:47:12,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:13,313 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:15,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:17,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:19,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:25,847 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:47:25,847 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:47:31,007 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:47:31,008 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:47:31,008 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:47:31,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:47:32,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:33,322 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:35,322 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:36,226 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:47:37,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:39,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:40,980 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:47:40,981 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:47:51,046 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:47:51,046 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:47:51,047 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:47:51,329 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:47:52,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:53,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:55,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:56,116 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:47:56,116 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:47:57,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:47:59,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:06,304 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:48:11,087 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:48:11,088 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:48:11,088 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:48:11,252 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:48:11,253 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:48:11,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:48:12,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:13,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:15,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:17,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:19,342 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:26,404 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:48:26,404 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:48:31,118 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:48:31,119 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:48:31,119 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:48:31,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:48:32,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:33,349 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:35,350 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:36,381 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:48:37,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:39,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:41,541 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:48:41,542 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:48:51,153 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:48:51,154 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:48:51,154 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:48:51,357 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:48:52,358 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:53,358 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:55,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:56,670 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:48:56,670 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:48:57,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:48:59,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:06,459 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:49:11,189 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:49:11,190 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:49:11,190 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:49:11,366 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:49:11,920 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:49:11,921 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:49:12,367 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:13,367 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:15,368 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:17,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:19,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:27,056 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:49:27,057 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:49:31,228 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:49:31,229 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:49:31,229 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:49:31,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:49:32,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:33,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:35,376 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:36,534 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:49:37,377 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:39,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:42,189 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:49:42,189 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:49:51,266 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:49:51,267 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:49:51,267 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:49:51,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:49:52,383 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:53,383 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:55,384 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:57,350 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:49:57,351 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:49:57,385 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:49:59,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:06,611 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:50:11,303 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:50:11,303 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:50:11,303 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:50:11,391 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:50:12,391 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:12,480 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:50:12,481 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:50:13,392 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:15,393 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:17,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:19,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:27,648 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:50:27,649 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:50:31,339 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:50:31,340 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:50:31,340 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:50:31,399 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:50:32,400 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:33,400 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:35,401 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:36,688 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:50:37,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:39,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:42,784 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:50:42,784 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:50:51,378 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:50:51,378 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:50:51,378 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:50:51,407 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:50:52,408 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:53,408 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:55,409 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:57,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:50:57,915 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:50:57,915 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:50:59,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:06,766 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:51:11,413 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:51:11,414 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:51:11,414 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:51:11,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:51:12,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:13,109 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:51:13,109 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:51:13,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:15,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:17,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:19,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:28,251 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:51:28,252 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:51:31,452 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:51:31,452 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:51:31,453 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:51:32,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:51:32,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:33,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:35,426 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:36,844 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:51:37,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:39,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:43,382 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:51:43,383 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:51:51,491 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:51:51,492 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:51:51,492 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:51:52,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:51:52,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:53,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:55,435 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:57,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:51:58,516 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:51:58,516 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:51:59,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:06,921 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:52:11,530 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:52:11,531 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:52:11,531 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:52:12,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:52:12,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:13,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:13,655 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:52:13,656 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:52:15,444 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:17,445 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:19,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:28,814 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:52:28,815 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:52:31,569 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:52:31,569 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:52:31,570 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:52:32,452 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:52:32,452 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:33,452 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:35,453 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:36,998 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:52:37,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:39,455 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:43,949 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:52:43,950 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:52:51,608 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:52:51,608 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:52:51,609 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:52:52,461 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:52:52,461 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:53,461 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:55,462 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:57,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:52:59,081 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:52:59,082 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:52:59,464 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:07,076 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:53:11,644 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:53:11,644 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:53:11,645 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:53:12,469 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:53:12,470 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:13,470 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:14,218 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:53:14,218 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:53:15,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:17,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:19,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:29,354 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:53:29,354 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:53:31,680 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:53:31,680 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:53:31,681 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:53:32,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:53:32,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:33,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:35,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:37,153 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:53:37,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:39,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:44,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:53:44,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:53:51,717 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:53:51,718 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:53:51,718 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:53:52,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:53:53,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:55,487 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:57,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:59,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:53:59,616 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:53:59,617 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:54:07,231 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:54:11,755 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:54:11,755 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:54:11,756 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:54:12,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:54:13,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:14,749 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:54:14,749 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:54:15,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:17,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:19,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:29,882 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:54:29,882 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:54:31,794 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:54:31,795 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:54:31,795 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:54:32,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:54:33,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:34,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:36,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:37,309 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:54:38,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:40,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:45,012 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:54:45,013 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:54:51,832 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:54:51,833 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:54:51,833 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:54:52,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:54:53,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:54,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:56,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:54:58,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:00,145 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:55:00,145 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:55:00,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:07,387 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:55:11,870 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:55:11,870 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:55:11,871 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:55:12,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:55:13,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:14,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:15,279 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:55:15,279 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:55:16,521 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:18,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:20,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:30,429 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:55:30,429 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:55:31,907 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:55:31,908 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:55:31,908 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:55:32,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:55:33,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:34,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:36,530 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:37,464 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:55:38,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:40,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:45,562 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:55:45,562 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:55:51,943 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:55:51,944 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:55:51,944 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:55:52,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:55:53,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:54,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:56,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:55:58,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:00,540 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:00,700 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:56:00,700 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:56:07,543 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:56:11,986 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:56:11,986 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:56:11,987 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:56:12,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:56:13,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:14,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:15,829 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:56:15,829 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:56:16,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:18,548 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:20,549 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:30,957 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:56:30,958 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:56:32,025 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:56:32,025 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:56:32,026 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:56:32,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:56:33,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:34,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:36,556 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:37,620 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:56:38,557 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:40,557 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:46,091 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:56:46,091 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:56:52,065 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:56:52,066 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:56:52,066 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:56:52,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:56:53,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:54,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:56,564 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:56:58,565 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:00,566 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:01,228 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:57:01,228 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:57:07,697 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:57:12,106 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:57:12,107 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:57:12,107 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:57:12,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:57:13,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:14,572 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:16,365 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:57:16,366 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:57:16,573 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:18,574 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:20,575 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:31,500 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:57:31,501 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:57:32,143 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:57:32,143 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:57:32,144 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:57:32,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:57:33,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:34,581 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:36,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:37,775 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:57:38,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:40,583 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:46,632 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:57:46,633 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:57:52,182 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:57:52,182 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:57:52,183 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:57:52,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:57:53,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:54,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:56,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:57:58,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:00,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:01,777 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:58:01,777 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:58:07,855 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:58:12,219 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:58:12,219 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:58:12,219 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:58:12,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:58:13,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:14,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:16,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:16,915 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:58:16,915 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:58:18,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:20,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:32,045 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:58:32,046 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:58:32,259 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:58:32,259 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:58:32,259 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:58:32,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:58:33,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:34,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:36,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:37,930 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:58:38,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:40,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:47,178 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:58:47,178 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:58:52,297 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:58:52,297 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:58:52,298 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:58:52,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:58:53,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:54,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:56,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:58:58,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:00,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:02,309 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:59:02,310 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:59:08,008 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:59:12,334 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:59:12,334 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:59:12,334 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:59:12,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:59:13,624 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:14,624 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:16,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:17,458 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:59:17,459 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:59:18,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:20,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:32,370 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:59:32,370 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:59:32,371 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:59:32,620 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:59:32,621 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:59:32,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:59:33,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:34,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:36,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:38,086 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 14:59:38,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:40,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:47,790 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 14:59:47,791 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 14:59:52,409 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 14:59:52,410 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 14:59:52,410 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 14:59:52,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 14:59:53,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:54,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:56,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 14:59:58,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:00,646 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:02,924 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:00:02,924 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:00:08,164 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:00:12,675 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:00:12,675 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:00:12,676 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:00:13,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:00:13,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:14,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:16,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:18,056 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:00:18,057 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:00:18,654 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:20,655 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:32,712 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:00:32,712 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:00:32,713 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:00:33,194 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:00:33,195 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:00:33,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:00:33,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:34,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:36,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:38,242 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:00:38,663 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:40,664 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:48,333 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:00:48,334 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:00:52,748 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:00:52,748 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:00:52,749 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:00:53,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:00:53,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:54,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:56,671 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:00:58,672 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:00,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:03,467 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:01:03,467 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:01:08,320 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:01:12,781 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:01:12,782 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:01:12,782 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:01:13,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:01:13,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:14,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:16,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:18,602 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:01:18,602 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:01:18,680 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:20,681 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:32,819 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:01:32,819 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:01:32,819 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:01:33,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:01:33,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:33,737 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:01:33,738 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:01:34,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:36,688 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:38,397 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:01:38,689 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:40,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:48,887 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:01:48,887 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:01:52,854 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:01:52,855 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:01:52,855 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:01:53,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:01:53,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:54,696 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:56,696 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:01:58,697 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:00,698 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:04,018 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:02:04,018 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:02:08,475 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:02:12,891 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:02:12,891 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:02:12,892 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:02:13,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:02:13,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:14,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:16,705 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:18,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:19,150 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:02:19,151 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:02:20,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:32,928 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:02:32,929 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:02:32,929 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:02:33,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:02:33,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:34,315 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:02:34,315 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:02:34,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:36,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:38,552 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:02:38,714 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:40,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:49,453 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:02:49,453 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:02:52,962 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:02:52,962 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:02:52,963 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:02:53,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:02:53,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:54,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:56,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:02:58,722 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:00,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:04,586 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:03:04,587 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:03:08,631 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:03:12,997 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:03:12,997 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:03:12,998 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:03:13,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:03:13,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:14,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:16,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:18,730 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:19,717 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:03:19,717 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:03:20,731 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:33,032 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:03:33,032 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:03:33,033 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:03:33,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:03:33,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:34,877 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:03:34,877 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:03:35,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:37,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:38,708 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:03:39,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:41,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:50,044 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:03:50,044 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:03:53,070 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:03:53,071 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:03:53,071 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:03:53,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:03:53,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:55,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:57,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:03:59,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:01,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:05,178 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:04:05,179 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:04:08,786 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:04:13,108 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:04:13,109 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:04:13,109 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:04:13,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:04:14,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:15,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:17,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:19,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:20,312 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:04:20,312 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:04:21,756 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:33,147 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:04:33,147 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:04:33,148 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:04:33,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:04:34,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:35,605 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:04:35,605 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:04:35,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:37,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:38,864 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:04:39,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:41,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:50,746 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:04:50,747 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:04:53,185 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:04:53,186 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:04:53,186 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:04:53,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:04:54,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:55,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:57,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:04:59,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:01,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:05,889 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:05:05,889 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:05:08,942 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:05:13,222 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:05:13,222 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:05:13,222 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:05:13,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:05:14,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:15,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:17,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:19,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:21,021 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:05:21,021 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:05:21,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:33,259 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:05:33,259 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:05:33,260 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:05:33,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:05:34,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:35,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:36,160 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:05:36,160 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:05:37,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:39,018 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:05:39,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:41,791 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:51,290 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:05:51,290 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:05:53,298 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:05:53,298 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:05:53,299 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:05:53,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:05:54,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:55,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:57,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:05:59,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:01,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:06,424 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:06:06,424 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:06:09,098 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:06:13,336 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:06:13,337 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:06:13,337 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:06:13,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:06:14,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:15,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:17,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:19,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:21,556 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:06:21,556 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:06:21,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:33,374 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:06:33,375 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:06:33,375 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:06:33,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:06:34,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:35,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:36,687 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:06:36,688 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:06:37,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:39,175 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:06:39,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:41,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:51,821 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:06:51,821 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:06:53,411 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:06:53,411 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:06:53,412 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:06:53,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:06:54,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:55,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:57,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:06:59,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:01,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:06,979 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:07:06,980 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:07:09,253 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:07:13,447 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:07:13,448 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:07:13,448 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:07:13,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:07:14,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:15,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:17,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:19,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:21,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:22,111 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:07:22,112 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:07:33,486 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:07:33,486 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:07:33,486 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:07:33,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:07:34,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:35,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:37,242 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:07:37,242 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:07:37,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:39,331 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:07:39,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:41,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:52,381 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:07:52,382 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:07:53,523 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:07:53,524 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:07:53,524 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:07:53,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:07:54,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:55,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:57,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:07:59,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:01,853 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:07,515 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:08:07,516 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:08:09,411 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:08:13,561 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:08:13,562 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:08:13,562 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:08:13,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:08:14,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:15,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:17,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:19,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:21,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:22,645 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:08:22,646 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:08:33,600 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:08:33,600 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:08:33,601 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:08:33,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:08:34,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:35,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:37,775 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:08:37,776 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:08:37,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:39,492 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:08:39,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:41,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:52,907 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:08:52,908 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:08:53,638 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:08:53,638 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:08:53,638 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:08:53,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:08:54,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:55,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:57,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:08:59,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:01,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:08,039 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:09:08,040 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:09:09,568 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:09:13,678 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:09:13,678 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:09:13,679 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:09:13,884 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:09:14,885 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:15,885 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:17,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:19,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:21,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:23,173 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:09:23,173 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:09:33,715 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:09:33,716 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:09:33,716 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:09:33,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:09:34,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:35,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:37,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:38,305 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:09:38,306 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:09:39,646 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:09:39,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:41,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:53,436 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:09:53,436 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:09:53,752 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:09:53,752 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:09:53,753 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:09:53,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:09:54,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:55,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:57,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:09:59,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:01,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:08,568 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:10:08,569 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:10:09,725 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:10:13,789 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:10:13,789 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:10:13,790 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:10:13,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:10:14,911 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:15,911 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:17,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:19,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:21,914 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:23,705 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:10:23,705 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:10:33,824 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:10:33,824 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:10:33,825 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:10:33,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:10:34,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:35,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:37,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:38,836 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:10:38,836 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:10:39,803 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:10:39,922 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:41,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:53,863 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:10:53,863 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:10:53,864 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:10:53,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:10:53,983 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:10:53,983 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:10:54,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:55,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:57,930 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:10:59,931 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:01,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:09,121 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:11:09,122 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:11:09,879 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:11:13,900 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:11:13,900 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:11:13,901 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:11:13,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:11:14,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:15,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:17,939 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:19,939 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:21,940 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:24,253 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:11:24,254 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:11:33,938 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:11:33,939 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:11:33,940 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:11:33,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:11:34,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:35,947 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:37,948 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:39,384 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:11:39,385 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:11:39,949 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:39,961 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:11:41,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:53,976 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:11:53,976 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:11:53,977 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:11:54,518 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:11:54,518 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:11:54,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:11:54,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:55,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:57,956 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:11:59,957 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:01,958 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:09,654 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:12:09,655 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:12:10,034 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:12:14,013 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:12:14,013 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:12:14,014 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:12:14,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:12:14,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:15,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:17,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:19,965 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:21,966 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:24,789 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:12:24,789 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:12:34,051 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:12:34,052 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:12:34,052 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:12:34,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:12:34,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:35,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:37,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:39,920 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:12:39,920 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:12:39,974 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:40,112 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:12:41,975 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:54,090 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:12:54,091 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:12:54,091 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:12:54,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:12:55,056 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:12:55,057 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:12:55,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:57,982 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:12:59,983 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:01,984 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:10,189 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:13:10,189 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:13:10,192 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:13:14,128 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:13:14,129 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:13:14,129 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:13:14,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:13:15,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:16,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:18,991 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:20,992 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:22,993 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:25,322 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:13:25,322 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:13:34,168 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:13:34,169 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:13:34,169 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:13:34,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:13:36,000 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:37,000 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:39,001 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:40,268 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:13:40,458 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:13:40,459 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:13:41,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:43,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:54,204 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:13:54,205 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:13:54,205 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:13:55,009 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:13:55,606 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:13:55,606 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:13:56,009 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:57,010 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:13:59,011 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:01,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:03,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:10,346 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:14:10,747 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:14:10,747 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:14:14,242 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:14:14,242 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:14:14,243 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:14:15,018 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:14:16,018 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:17,019 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:19,020 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:21,021 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:23,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:25,878 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:14:25,878 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:14:34,282 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:14:34,282 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:14:34,283 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:14:35,027 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:14:36,028 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:37,028 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:39,029 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:40,422 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:14:41,030 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:41,180 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:14:41,180 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:14:43,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:54,323 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:14:54,323 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:14:54,324 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:14:55,036 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:14:56,036 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:56,332 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:14:56,333 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:14:57,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:14:59,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:01,038 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:03,039 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:10,501 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:15:11,468 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:15:11,469 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:15:14,360 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:15:14,360 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:15:14,361 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:15:15,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:15:16,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:17,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:19,046 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:21,047 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:23,048 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:26,613 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:15:26,614 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:15:34,396 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:15:34,397 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:15:34,398 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:15:35,053 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:15:36,053 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:37,053 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:39,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:40,579 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:15:41,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:41,750 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:15:41,750 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:15:43,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:54,432 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:15:54,432 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:15:54,433 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:15:55,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:15:56,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:56,883 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:15:56,883 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:15:57,062 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:15:59,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:01,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:03,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:10,656 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:16:12,027 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:16:12,028 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:16:14,473 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:16:14,474 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:16:14,474 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:16:15,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:16:16,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:17,070 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:19,071 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:21,071 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:23,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:27,159 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:16:27,159 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:16:34,748 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:16:34,749 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:16:34,749 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:16:35,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:16:36,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:37,079 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:39,080 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:40,734 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:16:41,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:42,291 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:16:42,292 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:16:43,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:54,785 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:16:54,785 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:16:54,786 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:16:55,087 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:16:56,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:57,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:16:57,424 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:16:57,425 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:16:59,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:01,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:03,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:10,811 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:17:12,558 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:17:12,558 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:17:14,822 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:17:14,822 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:17:14,823 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:17:15,095 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:17:16,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:17,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:19,097 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:21,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:23,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:27,690 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:17:27,690 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:17:34,860 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:17:34,861 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:17:34,861 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:17:35,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:17:36,105 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:37,105 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:39,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:40,890 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:17:41,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:42,822 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:17:42,822 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:17:43,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:54,896 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:17:54,897 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:17:54,897 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:17:55,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:17:56,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:57,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:17:57,958 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:17:57,958 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:17:59,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:01,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:03,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:10,967 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:18:13,094 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:18:13,094 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:18:14,934 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:18:14,935 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:18:14,935 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:18:15,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:18:16,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:17,122 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:19,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:21,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:23,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:28,225 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:18:28,226 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:18:34,975 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:18:34,975 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:18:34,976 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:18:35,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:18:36,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:37,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:39,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:41,046 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:18:41,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:43,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:43,360 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:18:43,360 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:18:55,013 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:18:55,014 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:18:55,014 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:18:55,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:18:56,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:57,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:18:58,492 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:18:58,492 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:18:59,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:01,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:03,144 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:11,122 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:19:13,624 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:19:13,625 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:19:15,049 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:19:15,049 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:19:15,050 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:19:15,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:19:16,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:17,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:19,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:21,152 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:23,153 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:28,762 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:19:28,762 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:19:35,086 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:19:35,086 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:19:35,087 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:19:35,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:19:36,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:37,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:39,160 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:41,161 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:41,200 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:19:43,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:43,895 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:19:43,895 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:19:55,125 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:19:55,125 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:19:55,126 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:19:55,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:19:56,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:57,168 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:19:59,027 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:19:59,027 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:19:59,169 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:01,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:03,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:11,277 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:20:14,158 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:20:14,158 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:20:15,163 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:20:15,164 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:20:15,164 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:20:15,176 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:20:16,176 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:17,177 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:19,177 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:21,178 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:23,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:29,290 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:20:29,290 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:20:35,200 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:20:35,200 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:20:35,201 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:20:36,185 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:20:36,185 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:37,186 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:39,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:41,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:41,356 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:20:43,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:44,431 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:20:44,432 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:20:45,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:47,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:49,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:51,193 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:53,194 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:55,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:57,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:59,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:20:59,566 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:20:59,567 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:21:02,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:04,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:06,200 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:08,202 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:10,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:11,433 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:21:12,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:14,205 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:14,700 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:21:14,700 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:21:16,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:18,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:20,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:22,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:24,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:26,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:28,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:29,834 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:21:29,834 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:21:30,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:32,214 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:34,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:36,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:39,061 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:21:39,061 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:21:39,062 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:21:39,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:21:41,512 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:21:45,145 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:21:45,146 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:21:48,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:50,223 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:21:52,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:22:00,300 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:22:00,300 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status