diff --git "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" --- "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" +++ "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" @@ -8074,3 +8074,995 @@ 2021-07-15 22:46:00,146 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status 2021-07-15 22:46:00,147 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status 2021-07-15 22:46:00,285 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:10,266 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:46:10,266 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:46:10,267 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:46:10,289 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:46:11,289 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:12,290 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:14,291 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:15,283 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:46:15,283 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:46:16,292 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:18,293 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:20,294 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:21,396 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:46:30,305 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:46:30,305 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:46:30,305 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:46:30,639 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:46:30,640 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:46:31,298 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:46:31,299 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:32,299 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:34,300 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:36,301 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:38,302 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:40,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:45,779 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:46:45,779 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:46:50,341 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:46:50,342 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:46:50,342 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:46:51,308 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:46:51,308 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:51,472 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:46:52,308 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:54,309 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:56,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:46:58,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:00,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:00,922 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:47:00,923 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:47:10,378 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:47:10,379 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:47:10,379 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:47:11,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:47:11,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:12,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:14,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:16,074 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:47:16,074 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:47:16,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:18,320 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:21,543 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:47:30,422 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:47:30,422 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:47:30,423 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:47:31,206 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:47:31,206 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:47:31,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:47:31,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:32,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:34,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:36,327 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:38,328 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:46,349 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:47:46,349 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:47:50,466 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:47:50,467 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:47:50,467 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:47:51,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:47:51,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:51,611 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:47:52,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:54,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:56,336 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:47:58,337 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:01,479 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:48:01,479 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:48:10,511 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:48:10,512 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:48:10,512 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:48:11,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:48:11,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:12,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:14,343 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:16,344 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:17,399 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:48:17,400 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:48:18,345 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:20,346 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:21,685 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:48:30,550 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:48:30,550 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:48:30,551 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:48:31,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:48:31,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:32,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:32,557 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:48:32,558 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:48:34,352 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:36,353 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:38,354 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:40,355 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:47,697 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:48:47,697 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:48:50,637 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:48:50,638 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:48:50,638 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:48:51,359 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:48:51,763 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:48:52,360 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:54,361 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:56,362 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:48:58,363 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:02,835 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:49:02,835 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:49:10,677 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:49:10,678 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:49:10,678 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:49:11,368 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:49:12,368 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:14,369 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:16,370 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:17,967 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:49:17,967 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:49:18,371 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:21,849 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:49:30,714 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:49:30,714 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:49:30,715 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:49:31,375 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:49:32,376 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:33,124 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:49:33,125 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:49:34,377 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:36,378 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:38,378 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:48,272 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:49:48,272 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:49:50,748 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:49:50,749 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:49:50,749 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:49:51,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:49:51,929 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:49:52,384 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:54,385 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:56,386 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:49:58,386 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:00,387 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:03,408 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:50:03,408 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:50:10,786 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:50:10,786 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:50:10,787 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:50:11,391 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:50:12,392 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:14,393 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:16,394 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:18,394 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:18,539 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:50:18,539 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:50:20,395 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:22,007 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:50:30,825 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:50:30,825 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:50:30,826 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:50:31,399 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:50:32,400 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:33,687 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:50:33,687 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:50:34,401 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:36,402 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:38,403 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:40,403 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:48,818 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:50:48,819 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:50:50,864 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:50:50,864 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:50:50,865 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:50:51,408 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:50:52,083 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:50:52,408 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:54,409 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:56,410 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:50:58,410 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:00,411 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:02,412 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:03,960 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:51:03,960 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:51:04,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:06,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:08,414 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:10,415 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:12,415 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:14,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:16,417 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:18,418 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:19,091 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:51:19,091 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:51:20,419 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:22,158 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:51:23,420 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:25,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:27,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:29,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:31,423 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:33,423 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:34,229 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:51:34,229 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:51:35,424 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:37,425 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:39,426 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:41,426 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:43,427 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:45,428 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:47,428 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:49,384 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:51:49,385 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:51:49,429 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:51,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:52,226 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:51:53,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:54,713 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:51:54,713 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:51:54,713 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:51:55,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:51:57,432 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:51:59,433 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:01,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:03,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:04,519 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:52:04,520 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:52:14,759 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:52:14,759 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:52:14,760 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:52:15,438 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:52:16,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:17,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:19,440 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:19,650 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:52:19,651 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:52:21,441 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:22,291 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:52:23,441 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:34,784 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:52:34,785 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:52:34,915 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:52:34,915 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:52:34,915 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:52:35,445 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:52:36,446 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:37,446 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:39,447 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:41,448 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:43,448 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:49,919 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:52:49,919 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:52:52,355 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:52:54,849 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:52:54,849 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:52:54,849 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:52:55,453 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:52:55,453 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:56,453 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:57,454 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:52:59,454 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:01,455 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:03,456 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:05,050 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:53:05,050 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:53:14,905 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:53:14,905 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:53:14,906 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:53:15,460 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:53:16,461 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:17,461 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:19,462 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:20,194 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:53:20,194 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:53:21,463 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:22,419 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:53:23,464 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:34,940 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:53:34,941 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:53:34,941 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:53:35,327 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:53:35,328 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:53:35,468 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:53:36,469 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:37,469 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:39,470 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:41,471 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:43,472 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:50,488 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:53:50,489 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:53:52,486 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:53:54,977 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:53:54,977 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:53:54,978 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:53:55,476 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:53:56,477 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:57,477 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:53:59,478 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:01,479 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:03,480 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:05,622 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:54:05,622 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:54:15,031 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:54:15,032 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:54:15,032 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:54:15,484 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:54:17,485 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:19,486 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:20,752 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:54:20,752 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:54:21,487 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:22,555 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:54:23,488 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:35,068 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:54:35,068 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:54:35,068 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:54:35,492 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:54:35,918 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:54:35,918 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:54:37,493 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:39,494 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:41,495 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:43,495 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:51,060 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:54:51,061 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:54:52,625 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:54:55,119 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:54:55,119 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:54:55,119 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:54:55,500 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:54:57,501 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:54:59,502 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:01,503 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:03,503 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:06,191 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:55:06,192 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:55:15,154 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:55:15,155 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:55:15,155 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:55:15,507 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:55:17,508 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:19,509 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:21,325 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:55:21,326 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:55:21,510 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:22,696 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:55:23,511 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:35,226 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:55:35,227 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:55:35,227 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:55:35,515 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:55:36,462 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:55:36,463 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:55:37,516 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:39,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:41,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:43,518 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:51,626 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:55:51,626 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:55:52,771 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:55:55,266 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:55:55,266 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:55:55,266 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:55:55,522 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:55:55,522 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:57,523 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:55:59,524 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:01,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:03,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:06,756 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:56:06,756 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:56:15,333 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:56:15,333 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:56:15,334 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:56:15,530 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:56:17,530 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:18,531 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:20,531 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:21,892 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:56:21,892 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:56:22,532 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:22,843 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:56:24,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:35,373 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:56:35,373 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:56:35,374 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:56:35,537 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:56:37,045 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:56:37,045 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:56:37,538 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:38,538 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:40,539 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:42,540 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:44,541 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:52,184 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:56:52,184 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:56:52,916 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:56:55,410 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:56:55,410 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:56:55,411 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:56:55,545 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:56:57,546 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:56:58,547 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:00,547 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:02,548 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:04,549 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:07,318 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:57:07,318 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:57:15,467 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:57:15,468 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:57:15,468 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:57:15,552 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:57:17,553 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:18,554 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:20,555 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:22,453 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:57:22,454 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:57:22,555 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:22,989 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:57:24,556 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:35,501 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:57:35,501 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:57:35,502 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:57:35,560 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:57:36,560 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:37,560 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:37,618 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:57:37,619 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:57:38,561 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:40,562 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:42,562 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:44,563 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:52,750 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:57:52,751 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:57:53,059 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:57:55,542 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:57:55,542 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:57:55,543 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:57:55,566 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:57:56,567 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:57,567 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:57:58,567 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:00,568 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:02,569 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:04,569 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:07,881 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:58:07,881 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:58:15,582 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:58:15,582 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:58:15,583 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:58:16,573 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:58:16,574 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:17,574 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:18,574 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:20,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:22,576 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:23,013 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:58:23,013 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:58:23,144 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:58:24,577 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:35,619 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:58:35,620 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:58:35,620 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:58:36,581 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:58:36,581 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:37,581 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:38,160 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:58:38,161 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:58:38,582 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:40,583 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:42,584 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:44,585 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:53,207 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:58:53,292 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:58:53,292 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:58:55,660 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:58:55,661 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:58:55,661 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:58:56,589 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:58:56,589 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:57,590 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:58:58,590 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:00,591 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:02,592 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:04,593 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:08,426 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:59:08,426 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:59:15,704 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:59:15,704 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:59:15,705 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:59:16,596 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:59:16,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:17,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:18,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:20,598 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:22,599 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:23,281 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:59:23,555 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:59:23,556 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:59:24,600 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:35,741 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:59:35,742 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:59:35,742 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:59:36,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:59:36,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:37,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:38,605 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:38,683 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:59:38,684 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:59:40,605 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:42,606 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:44,607 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:53,359 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:59:53,814 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:59:53,815 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:59:55,792 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:59:55,793 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:59:55,793 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:59:56,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:59:56,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:57,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:59:58,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:00,613 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:02,613 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:04,614 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:08,945 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:00:08,945 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:00:15,832 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:00:15,833 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:00:15,833 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:00:16,618 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:00:16,618 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:17,619 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:18,619 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:20,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:22,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:23,433 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:00:24,075 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:00:24,076 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:00:24,621 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:35,878 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:00:35,878 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:00:35,879 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:00:36,625 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:00:36,626 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:37,626 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:38,626 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:39,211 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:00:39,212 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:00:40,627 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:42,628 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:44,628 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:53,507 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:00:54,348 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:00:54,349 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:00:55,932 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:00:55,932 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:00:55,933 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:00:56,633 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:00:56,633 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:57,633 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:00:58,634 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:00,634 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:02,635 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:04,636 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:09,479 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:01:09,479 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:01:15,976 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:01:15,976 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:01:15,977 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:01:16,640 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:01:17,641 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:19,641 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:21,642 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:23,582 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:01:23,643 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:24,616 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:01:24,617 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:01:25,643 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:36,014 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:01:36,014 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:01:36,015 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:01:36,648 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:01:37,648 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:39,649 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:39,749 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:01:39,749 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:01:41,649 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:43,650 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:45,651 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:53,659 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:01:54,879 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:01:54,879 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:01:56,051 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:01:56,052 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:01:56,052 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:01:56,655 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:01:57,655 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:01:59,656 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:01,657 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:03,658 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:05,659 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:10,012 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:02:10,013 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:02:16,114 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:02:16,115 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:02:16,115 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:02:16,662 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:02:17,663 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:19,664 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:21,665 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:23,666 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:23,732 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:02:25,145 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:02:25,145 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:02:25,666 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:36,155 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:02:36,155 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:02:36,156 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:02:36,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:02:37,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:39,671 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:40,281 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:02:40,282 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:02:41,672 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:43,673 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:45,674 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:53,805 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:02:55,415 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:02:55,415 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:02:56,193 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:02:56,193 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:02:56,194 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:02:56,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:02:57,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:02:59,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:01,680 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:03,680 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:05,681 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:10,547 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:03:10,547 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:03:16,231 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:03:16,231 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:03:16,232 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:03:16,684 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:03:17,685 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:19,686 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:21,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:23,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:23,877 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:03:25,679 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:03:25,679 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:03:25,688 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:36,267 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:03:36,267 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:03:36,268 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:03:36,692 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:03:37,692 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:39,693 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:40,809 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:03:40,810 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:03:41,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:43,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:45,695 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:53,953 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:03:55,942 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:03:55,943 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:03:56,304 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:03:56,304 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:03:56,305 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:03:56,699 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:03:57,699 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:03:59,700 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:01,700 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:03,701 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:05,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:11,073 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:04:11,074 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:04:16,342 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:04:16,342 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:04:16,342 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:04:16,705 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:04:17,706 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:18,706 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:19,706 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:21,707 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:23,708 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:24,025 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:04:25,709 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:26,203 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:04:26,204 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:04:36,376 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:04:36,377 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:04:36,377 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:04:36,713 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:04:37,713 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:38,714 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:39,714 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:41,334 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:04:41,335 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:04:41,715 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:43,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:45,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:54,101 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:04:56,414 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:04:56,414 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:04:56,415 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:04:56,465 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:04:56,466 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:04:56,720 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:04:57,721 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:58,721 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:04:59,722 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:01,722 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:03,723 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:05,724 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:11,597 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:05:11,598 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:05:16,458 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:05:16,459 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:05:16,459 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:05:16,728 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:05:17,729 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:18,729 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:19,730 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:21,731 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:23,731 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:24,177 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:05:25,732 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:26,730 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:05:26,730 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:05:36,504 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:05:36,504 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:05:36,505 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:05:36,737 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:05:37,737 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:38,737 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:39,738 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:41,739 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:41,862 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:05:41,862 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:05:43,740 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:45,741 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:54,253 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:05:56,541 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:05:56,541 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:05:56,545 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:05:56,745 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:05:57,002 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:05:57,002 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:05:57,745 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:58,746 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:05:59,746 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:01,747 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:03,747 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:05,748 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:12,186 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:06:12,186 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:06:16,577 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:06:16,577 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:06:16,578 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:06:16,752 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:06:17,752 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:18,752 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:19,753 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:21,753 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:23,754 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:24,329 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:06:25,755 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:27,316 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:06:27,316 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:06:36,645 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:06:36,646 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:06:36,646 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:06:36,758 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:06:37,759 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:38,759 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:39,760 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:41,761 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:42,445 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:06:42,445 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:06:43,762 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:45,763 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:54,401 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:06:56,680 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:06:56,681 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:06:56,681 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:06:56,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:06:57,613 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:06:57,613 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:06:57,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:58,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:06:59,768 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:01,768 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:03,769 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:05,770 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:12,752 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:07:12,752 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:07:16,718 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:07:16,719 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:07:16,719 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:07:16,774 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:07:17,774 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:18,775 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:19,775 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:21,776 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:23,777 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:24,474 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:07:25,777 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:27,885 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:07:27,885 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:07:36,809 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:07:36,809 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:07:36,810 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:07:37,781 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:07:37,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:38,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:39,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:41,783 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:43,078 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:07:43,078 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:07:43,784 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:45,785 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:54,551 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:07:56,849 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:07:56,850 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:07:56,850 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:07:57,789 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:07:57,789 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:58,211 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:07:58,211 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:07:58,790 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:07:59,790 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:01,791 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:03,792 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:05,793 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:13,344 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:08:13,344 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:08:16,910 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:08:16,911 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:08:16,911 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:08:17,797 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:08:17,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:18,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:19,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:21,799 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:23,800 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:24,625 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:08:25,801 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:28,482 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:08:28,482 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:08:36,956 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:08:36,956 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:08:36,956 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:08:37,805 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:08:37,805 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:38,806 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:39,806 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:41,807 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:43,614 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:08:43,615 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:08:43,808 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:45,808 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:54,699 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:08:57,063 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:08:57,064 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:08:57,064 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:08:57,812 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:08:57,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:58,751 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:08:58,751 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:08:58,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:08:59,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:01,814 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:04,815 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:06,816 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:13,882 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:09:13,883 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:09:17,102 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:09:17,102 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:09:17,103 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:09:17,820 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:09:18,820 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:20,821 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:22,822 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:24,771 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:09:24,822 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:26,823 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:29,013 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:09:29,013 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:09:37,155 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:09:37,155 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:09:37,156 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:09:37,827 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:09:38,827 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:40,828 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:42,828 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:44,144 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:09:44,144 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:09:44,829 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:46,830 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:54,846 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:09:57,193 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:09:57,193 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:09:57,194 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:09:57,834 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:09:58,834 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:09:59,283 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:09:59,283 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:10:00,835 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:02,836 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:04,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:06,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:14,413 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:10:14,413 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:10:17,229 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:10:17,230 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:10:17,230 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:10:17,841 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:10:18,841 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:20,842 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:22,843 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:24,844 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:24,920 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:10:26,845 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:29,545 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:10:29,546 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:10:37,265 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:10:37,265 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:10:37,265 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:10:37,849 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:10:38,850 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:40,851 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:42,852 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:44,675 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:10:44,676 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:10:44,852 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:46,853 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:54,994 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:10:57,301 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:10:57,301 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:10:57,301 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:10:57,857 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:10:58,857 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:10:59,810 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:10:59,810 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:11:00,858 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:02,859 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:04,860 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:06,861 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:14,984 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:11:14,985 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:11:17,346 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:11:17,347 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:11:17,347 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:11:17,865 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:11:18,865 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:20,866 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:22,867 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:24,868 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:25,069 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:11:26,868 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:30,116 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:11:30,116 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:11:37,398 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:11:37,398 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:11:37,398 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:11:37,872 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:11:38,873 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:40,873 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:42,874 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:44,875 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:45,249 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:11:45,249 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:11:46,876 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:11:55,144 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:11:57,434 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:11:57,435 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:11:57,435 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:11:57,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:11:58,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:12:00,426 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:12:00,427 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:12:05,883 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:12:07,884 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:12:09,885 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:12:15,616 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:12:15,616 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status