diff --git "a/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log" "b/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log" --- "a/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log" +++ "b/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log" @@ -7508,3 +7508,7448 @@ 2021-07-13 04:20:52,050 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status 2021-07-13 04:20:52,050 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status 2021-07-13 04:20:52,720 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:20:59,948 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:21:07,181 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:21:07,181 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:21:09,727 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:11,728 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:13,729 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:15,730 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:17,731 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:19,732 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:21,732 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:22,317 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:21:22,318 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:21:23,733 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:25,734 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:27,735 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:29,737 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:30,021 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:21:31,738 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:33,738 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:35,739 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:37,460 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:21:37,460 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:21:37,740 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:39,741 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:41,742 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:43,743 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:45,744 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:47,745 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:49,746 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:51,747 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:52,590 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:21:52,590 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:21:53,748 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:55,749 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:57,750 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:21:59,751 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:00,093 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:22:01,752 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:03,753 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:05,753 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:07,720 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:22:07,721 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:22:07,754 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:09,755 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:11,756 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:13,757 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:15,758 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:17,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:19,760 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:21,761 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:22,850 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:22:22,851 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:22:23,762 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:25,763 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:27,764 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:29,764 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:30,164 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:22:31,765 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:33,766 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:35,767 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:37,768 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:37,980 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:22:37,981 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:22:39,769 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:41,770 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:43,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:45,772 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:47,773 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:49,773 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:51,774 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:53,109 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:22:53,109 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:22:53,775 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:55,776 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:57,777 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:22:59,778 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:00,238 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:23:01,778 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:03,779 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:05,780 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:07,781 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:08,242 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:23:08,242 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:23:09,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:11,783 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:13,784 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:15,785 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:17,786 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:19,787 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:21,788 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:23,374 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:23:23,374 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:23:23,788 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:25,789 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:27,790 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:29,791 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:30,311 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:23:31,792 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:33,793 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:35,794 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:37,795 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:38,504 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:23:38,504 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:23:39,796 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:41,797 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:43,798 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:45,799 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:47,800 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:49,800 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:51,801 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:53,635 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:23:53,636 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:23:53,802 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:55,803 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:57,804 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:23:59,805 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:00,387 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:24:01,806 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:03,807 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:05,808 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:07,809 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:08,765 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:24:08,765 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:24:09,810 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:11,810 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:13,811 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:15,812 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:17,813 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:19,814 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:21,815 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:23,816 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:23,903 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:24:23,903 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:24:25,817 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:27,818 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:29,820 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:30,463 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:24:31,821 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:33,822 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:36,823 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:38,824 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:39,035 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:24:39,035 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:24:40,825 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:42,826 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:44,827 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:46,828 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:48,829 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:50,829 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:52,830 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:54,167 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:24:54,167 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:24:54,831 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:56,832 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:24:58,833 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:00,536 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:25:00,834 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:02,835 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:04,836 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:06,837 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:08,838 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:09,297 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:25:09,298 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:25:10,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:12,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:14,840 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:16,841 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:18,842 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:20,843 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:22,844 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:24,432 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:25:24,432 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:25:24,845 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:26,846 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:28,846 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:30,607 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:25:30,847 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:32,848 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:34,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:36,850 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:38,851 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:39,562 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:25:39,563 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:25:40,852 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:54,692 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:25:54,693 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:25:57,379 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:25:57,379 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:25:57,379 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:25:57,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:25:58,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:25:59,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:00,684 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:26:00,860 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:02,861 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:04,861 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:06,862 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:08,863 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:09,822 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:26:09,822 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:26:10,864 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:12,865 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:14,866 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:16,866 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:18,867 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:20,868 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:22,869 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:24,870 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:25,002 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:26:25,003 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:26:26,871 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:28,872 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:30,757 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:26:30,872 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:32,873 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:34,874 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:36,875 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:38,876 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:40,132 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:26:40,132 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:26:40,876 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:42,877 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:44,878 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:46,878 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:48,879 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:50,880 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:52,880 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:54,881 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:55,266 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:26:55,266 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:26:56,882 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:26:58,883 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:00,831 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:27:00,883 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:02,884 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:04,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:06,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:08,886 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:10,396 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:27:10,396 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:27:10,887 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:12,888 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:14,889 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:16,889 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:18,890 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:20,891 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:22,892 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:24,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:25,529 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:27:25,529 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:27:26,894 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:28,894 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:30,895 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:30,905 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:27:32,896 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:34,897 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:36,898 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:38,899 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:40,657 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:27:40,657 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:27:40,900 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:42,901 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:44,901 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:46,902 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:48,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:50,904 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:52,904 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:54,905 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:55,794 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:27:55,795 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:27:56,906 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:27:58,907 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:00,908 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:00,975 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:28:02,909 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:04,910 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:06,911 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:08,912 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:10,912 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:10,925 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:28:10,925 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:28:12,913 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:14,914 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:16,915 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:18,915 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:20,916 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:22,917 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:24,918 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:26,060 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:28:26,060 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:28:26,918 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:28,919 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:30,920 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:31,049 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:28:32,921 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:34,922 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:36,922 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:38,923 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:40,924 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:41,190 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:28:41,191 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:28:42,925 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:44,925 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:46,926 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:48,927 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:50,928 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:52,929 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:54,929 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:56,321 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:28:56,321 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:28:56,930 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:28:58,931 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:00,932 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:01,124 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:29:02,933 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:04,933 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:06,934 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:08,935 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:10,936 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:11,455 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:29:11,455 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:29:12,937 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:14,938 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:16,939 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:18,939 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:20,940 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:22,941 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:24,942 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:26,607 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:29:26,607 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:29:26,942 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:28,943 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:30,944 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:31,198 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:29:32,945 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:34,946 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:36,947 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:38,948 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:40,948 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:41,739 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:29:41,739 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:29:42,949 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:44,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:46,951 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:48,952 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:50,953 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:52,954 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:54,955 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:56,871 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:29:56,871 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:29:56,956 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:29:58,956 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:00,957 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:01,272 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:30:02,958 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:04,959 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:06,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:08,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:10,961 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:12,004 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:30:12,005 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:30:12,962 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:14,963 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:16,964 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:18,965 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:20,965 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:22,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:24,967 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:27,148 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:30:27,148 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:30:31,344 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:30:42,280 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:30:42,280 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:30:43,509 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:30:43,510 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:30:43,510 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:30:43,975 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:30:44,975 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:46,976 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:48,977 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:50,978 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:52,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:54,980 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:56,980 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:30:57,432 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:30:57,433 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:30:58,981 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:00,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:01,420 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:31:02,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:04,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:06,984 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:08,985 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:10,986 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:12,563 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:31:12,563 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:31:12,986 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:14,987 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:16,988 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:18,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:20,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:22,990 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:24,991 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:26,992 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:27,696 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:31:27,697 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:31:28,993 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:30,993 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:31,494 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:31:32,994 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:34,995 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:36,996 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:38,997 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:40,998 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:42,828 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:31:42,828 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:31:42,999 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:45,000 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:47,000 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:49,001 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:51,002 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:53,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:55,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:57,004 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:31:57,963 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:31:57,963 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:31:59,005 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:01,006 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:01,568 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:32:03,006 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:05,007 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:07,008 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:09,009 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:11,010 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:13,011 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:13,094 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:32:13,094 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:32:15,012 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:17,012 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:19,013 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:21,014 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:23,015 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:25,016 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:27,017 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:28,227 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:32:28,227 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:32:29,018 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:31,018 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:31,644 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:32:33,019 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:35,020 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:37,021 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:39,022 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:41,022 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:43,023 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:43,361 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:32:43,362 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:32:45,024 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:47,025 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:49,026 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:51,027 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:53,028 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:55,029 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:57,029 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:32:58,489 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:32:58,489 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:32:59,030 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:01,031 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:01,719 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:33:03,032 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:05,033 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:07,034 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:09,034 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:12,036 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:13,626 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:33:13,627 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:33:14,037 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:16,038 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:18,039 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:20,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:22,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:24,041 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:26,042 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:28,043 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:28,756 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:33:28,757 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:33:30,044 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:31,793 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:33:32,045 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:34,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:36,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:38,047 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:40,048 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:42,049 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:43,893 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:33:43,894 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:33:44,050 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:46,051 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:48,052 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:50,053 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:52,053 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:54,054 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:56,055 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:58,056 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:33:59,031 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:33:59,031 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:34:00,057 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:01,869 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:34:02,057 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:04,058 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:06,059 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:08,060 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:10,061 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:12,062 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:14,063 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:14,161 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:34:14,161 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:34:16,064 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:18,065 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:20,065 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:22,066 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:24,067 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:26,068 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:28,069 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:29,294 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:34:29,294 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:34:30,070 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:31,944 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:34:32,071 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:34,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:36,073 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:38,074 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:40,074 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:42,075 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:44,076 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:44,427 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:34:44,428 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:34:46,077 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:48,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:50,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:52,079 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:54,080 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:56,081 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:58,082 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:34:59,561 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:34:59,561 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:35:00,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:02,019 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:35:02,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:04,084 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:06,085 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:08,086 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:10,087 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:12,088 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:14,692 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:35:14,693 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:35:29,656 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:35:29,657 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:35:29,657 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:35:29,868 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:35:29,868 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:35:30,094 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:35:31,095 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:32,095 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:35:32,097 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:34,097 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:36,098 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:38,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:40,100 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:42,100 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:44,101 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:45,005 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:35:45,005 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:35:46,102 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:48,103 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:50,104 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:52,105 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:54,106 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:56,107 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:35:58,108 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:00,109 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:00,136 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:36:00,136 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:36:02,110 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:02,170 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:36:04,111 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:06,112 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:08,113 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:10,114 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:12,115 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:14,116 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:15,268 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:36:15,269 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:36:16,117 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:18,118 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:20,119 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:22,120 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:24,120 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:26,121 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:28,122 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:30,123 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:30,400 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:36:30,400 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:36:32,124 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:32,244 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:36:34,125 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:36,126 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:38,127 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:40,128 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:42,128 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:44,129 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:45,531 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:36:45,532 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:36:46,130 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:48,131 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:50,132 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:52,133 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:54,134 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:56,135 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:36:58,136 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:00,137 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:00,664 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:37:00,665 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:37:02,138 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:02,321 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:37:04,139 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:06,140 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:08,141 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:10,142 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:12,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:14,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:15,792 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:37:15,792 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:37:16,144 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:18,145 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:20,146 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:22,147 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:24,148 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:26,148 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:28,149 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:30,150 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:30,927 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:37:30,928 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:37:32,151 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:32,396 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:37:34,152 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:36,153 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:38,154 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:40,155 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:42,155 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:44,156 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:46,058 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:37:46,058 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:37:46,157 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:48,158 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:50,159 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:52,160 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:54,161 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:56,162 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:37:58,163 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:00,164 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:01,191 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:38:01,191 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:38:02,165 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:02,471 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:38:04,166 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:06,167 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:08,168 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:10,168 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:12,169 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:14,170 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:16,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:16,321 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:38:16,322 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:38:18,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:20,172 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:22,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:24,174 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:26,175 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:28,176 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:30,177 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:31,456 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:38:31,456 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:38:32,178 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:32,546 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:38:34,178 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:36,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:38,180 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:40,181 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:42,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:44,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:46,183 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:46,587 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:38:46,587 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:38:48,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:50,185 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:52,185 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:54,186 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:56,187 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:38:58,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:00,189 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:01,719 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:39:01,720 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:39:02,190 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:02,622 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:39:04,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:06,192 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:08,192 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:10,193 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:12,194 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:14,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:16,196 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:16,849 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:39:16,849 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:39:18,197 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:20,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:22,199 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:24,200 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:26,201 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:28,202 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:30,203 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:31,983 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:39:31,983 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:39:32,203 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:32,695 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:39:34,204 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:36,205 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:38,206 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:40,207 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:42,208 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:44,209 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:46,210 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:47,116 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:39:47,116 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:39:48,211 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:50,212 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:52,213 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:54,214 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:56,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:39:58,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:02,247 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:40:02,247 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:40:02,770 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:40:15,810 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:40:15,810 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:40:15,810 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:40:16,222 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:40:17,223 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:17,380 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:40:17,380 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:40:18,223 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:20,224 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:22,225 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:24,226 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:26,227 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:28,227 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:30,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:32,229 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:32,587 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:40:32,587 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:40:32,844 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:40:34,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:36,231 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:38,232 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:40,233 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:42,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:44,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:46,235 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:47,723 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:40:47,724 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:40:48,236 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:50,237 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:52,238 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:54,239 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:56,240 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:40:58,241 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:00,242 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:02,243 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:02,855 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:41:02,855 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:41:02,985 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:41:04,244 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:07,245 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:09,246 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:11,247 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:13,248 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:15,249 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:17,250 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:17,986 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:41:17,987 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:41:19,251 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:21,251 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:23,252 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:25,254 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:27,255 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:29,256 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:31,257 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:32,998 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:41:33,118 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:41:33,119 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:41:33,258 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:35,259 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:37,259 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:39,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:41,261 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:43,262 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:45,263 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:47,264 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:48,257 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:41:48,258 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:41:49,265 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:51,266 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:53,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:55,268 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:57,269 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:41:59,270 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:01,271 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:03,073 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:42:03,272 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:03,399 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:42:03,399 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:42:05,273 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:07,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:09,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:11,275 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:13,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:15,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:17,277 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:18,527 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:42:18,527 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:42:19,278 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:21,279 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:23,280 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:25,281 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:27,282 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:29,282 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:31,283 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:33,144 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:42:33,284 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:33,659 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:42:33,659 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:42:35,285 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:37,286 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:39,287 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:41,288 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:43,289 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:45,290 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:47,291 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:48,788 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:42:48,789 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:42:49,292 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:51,293 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:53,294 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:55,295 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:57,296 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:42:59,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:01,298 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:03,219 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:43:03,299 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:03,924 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:43:03,924 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:43:05,300 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:07,301 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:09,302 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:11,303 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:13,304 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:15,305 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:17,306 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:19,057 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:43:19,057 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:43:19,307 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:21,308 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:23,308 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:25,309 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:27,310 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:29,311 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:31,312 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:33,294 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:43:33,313 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:34,189 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:43:34,189 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:43:35,314 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:37,315 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:39,316 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:41,317 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:43,318 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:45,319 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:47,320 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:49,318 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:43:49,318 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:43:49,321 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:51,322 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:53,323 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:55,324 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:57,325 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:43:59,326 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:01,327 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:03,328 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:03,372 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:44:04,450 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:44:04,451 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:44:05,329 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:07,329 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:09,330 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:11,331 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:13,332 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:15,333 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:17,334 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:19,335 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:19,581 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:44:19,581 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:44:21,336 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:23,337 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:25,338 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:27,339 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:29,340 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:31,341 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:33,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:33,444 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:44:34,725 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:44:34,726 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:44:35,343 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:37,344 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:39,345 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:41,345 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:43,346 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:44:49,853 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:44:49,854 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:45:01,932 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:45:01,933 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:45:01,933 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:45:02,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:45:03,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:03,508 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:45:04,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:05,006 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:45:05,006 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:45:05,355 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:07,356 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:09,356 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:11,357 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:13,357 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:15,358 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:17,359 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:19,360 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:20,136 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:45:20,137 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:45:21,360 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:23,361 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:25,362 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:27,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:29,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:31,364 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:33,365 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:33,571 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:45:35,267 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:45:35,268 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:45:35,366 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:37,367 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:39,368 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:41,369 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:43,369 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:45,370 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:47,371 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:49,371 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:50,396 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:45:50,396 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:45:51,372 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:53,373 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:55,374 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:57,374 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:45:59,375 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:01,376 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:03,377 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:03,632 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:46:05,377 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:05,530 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:46:05,530 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:46:07,378 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:09,379 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:11,379 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:13,380 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:15,381 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:17,382 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:19,383 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:20,661 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:46:20,661 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:46:21,383 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:23,384 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:25,385 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:27,386 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:29,386 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:31,387 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:33,388 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:33,699 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:46:35,389 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:35,792 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:46:35,793 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:46:37,390 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:39,391 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:41,391 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:43,392 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:45,393 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:47,394 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:49,395 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:50,922 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:46:50,922 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:46:51,396 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:53,396 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:55,397 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:57,398 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:46:59,399 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:01,400 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:03,401 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:03,764 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:47:05,401 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:06,057 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:47:06,057 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:47:07,402 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:09,403 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:11,404 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:13,404 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:15,405 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:17,406 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:19,407 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:21,186 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:47:21,186 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:47:21,408 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:23,408 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:25,409 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:27,410 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:29,411 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:31,412 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:33,413 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:33,831 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:47:35,414 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:36,319 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:47:36,319 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:47:37,415 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:39,416 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:41,416 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:43,417 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:45,418 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:47,419 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:49,420 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:51,420 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:51,449 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:47:51,449 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:47:53,421 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:55,422 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:57,423 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:47:59,424 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:01,424 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:03,425 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:03,900 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:48:05,426 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:06,584 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:48:06,584 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:48:07,427 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:09,428 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:11,429 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:13,429 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:15,430 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:17,431 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:19,432 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:21,433 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:21,714 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:48:21,715 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:48:23,433 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:25,434 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:27,435 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:30,436 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:32,437 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:33,981 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:48:34,438 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:36,439 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:36,852 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:48:36,853 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:48:38,439 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:40,440 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:42,441 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:44,442 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:46,443 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:48,443 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:50,444 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:51,986 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:48:51,986 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:48:52,445 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:54,446 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:56,447 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:48:58,448 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:00,449 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:02,450 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:04,030 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:49:04,451 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:06,452 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:07,117 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:49:07,118 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:49:08,453 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:10,454 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:12,455 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:14,456 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:16,457 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:18,458 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:20,459 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:22,249 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:49:22,249 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:49:22,460 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:24,460 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:26,461 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:28,462 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:30,463 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:34,094 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:49:37,380 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:49:37,380 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:49:48,056 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:49:48,056 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:49:48,056 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:49:48,470 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:49:50,471 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:52,472 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:52,512 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:49:52,512 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:49:54,473 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:56,474 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:49:58,475 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:00,476 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:02,477 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:04,158 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:50:04,478 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:06,479 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:07,689 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:50:07,689 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:50:08,480 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:10,481 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:12,482 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:14,483 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:16,484 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:18,485 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:20,486 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:22,487 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:22,819 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:50:22,819 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:50:24,488 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:26,489 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:28,490 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:30,491 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:32,492 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:34,224 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:50:34,493 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:36,493 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:37,957 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:50:37,957 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:50:38,494 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:40,495 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:42,496 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:44,497 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:46,498 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:48,499 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:50,500 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:52,501 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:53,088 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:50:53,089 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:50:54,502 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:56,503 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:50:58,503 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:00,504 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:02,505 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:04,291 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:51:04,506 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:06,507 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:08,220 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:51:08,220 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:51:08,508 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:10,509 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:12,510 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:14,510 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:16,511 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:18,512 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:20,513 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:22,514 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:23,349 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:51:23,349 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:51:24,514 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:26,515 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:28,516 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:30,517 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:32,517 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:34,357 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:51:34,518 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:36,519 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:38,484 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:51:38,485 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:51:38,520 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:40,521 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:42,522 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:44,523 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:46,524 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:48,525 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:50,526 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:52,527 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:53,614 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:51:53,614 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:51:54,528 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:56,529 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:51:58,530 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:00,531 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:02,532 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:04,424 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:52:04,532 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:06,533 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:08,534 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:08,749 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:52:08,750 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:52:10,535 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:12,536 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:14,537 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:16,538 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:18,539 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:20,540 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:22,541 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:23,880 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:52:23,880 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:52:24,542 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:26,543 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:28,544 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:30,545 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:32,546 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:34,490 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:52:34,547 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:36,547 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:38,548 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:39,013 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:52:39,014 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:52:40,549 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:42,550 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:44,551 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:46,552 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:48,553 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:50,554 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:52,555 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:54,144 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:52:54,145 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:52:54,556 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:56,557 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:52:58,557 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:00,558 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:02,559 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:04,557 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:53:04,560 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:06,561 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:08,562 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:09,278 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:53:09,279 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:53:10,563 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:12,564 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:14,565 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:16,566 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:18,567 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:20,568 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:22,569 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:24,410 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:53:24,410 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:53:24,570 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:26,571 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:28,572 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:30,573 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:32,574 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:34,575 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:34,627 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:53:36,576 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:38,577 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:39,538 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:53:39,538 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:53:40,578 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:42,579 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:44,580 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:46,581 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:48,582 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:50,582 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:52,583 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:54,584 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:54,666 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:53:54,667 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:53:56,585 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:53:58,586 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:00,587 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:02,588 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:04,589 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:04,696 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:54:06,590 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:08,591 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:09,800 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:54:09,800 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:54:10,592 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:12,593 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:14,593 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:16,594 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:24,930 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:54:24,930 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:54:34,183 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:54:34,183 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:54:34,184 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:54:34,601 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:54:34,770 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:54:35,601 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:36,601 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:38,602 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:40,110 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:54:40,111 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:54:40,603 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:42,604 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:44,605 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:46,605 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:48,606 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:50,607 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:52,608 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:54,609 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:55,240 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:54:55,240 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:54:56,610 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:54:58,611 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:00,612 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:02,613 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:04,613 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:04,843 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:55:06,614 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:08,615 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:10,375 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:55:10,376 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:55:10,616 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:12,617 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:14,618 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:16,619 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:18,620 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:20,621 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:22,622 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:24,622 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:25,506 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:55:25,507 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:55:26,623 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:28,624 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:30,625 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:32,626 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:34,627 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:34,916 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:55:36,628 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:38,629 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:40,639 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:55:40,640 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:55:41,630 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:43,631 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:45,632 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:47,633 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:49,634 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:51,635 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:53,636 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:55,637 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:55,769 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:55:55,769 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:55:57,637 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:55:59,638 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:01,639 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:03,640 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:04,985 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:56:05,641 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:07,642 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:09,643 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:10,902 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:56:10,902 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:56:11,644 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:13,645 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:15,646 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:17,647 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:19,648 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:21,649 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:23,650 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:25,651 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:26,032 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:56:26,033 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:56:27,652 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:29,653 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:31,654 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:33,655 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:35,060 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:56:35,656 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:37,657 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:39,658 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:41,183 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:56:41,183 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:56:41,659 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:43,660 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:45,661 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:47,662 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:49,663 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:51,664 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:53,665 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:55,666 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:56,314 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:56:56,314 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:56:57,667 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:56:59,668 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:01,669 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:03,670 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:05,134 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:57:05,671 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:07,672 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:09,672 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:11,450 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:57:11,450 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:57:11,673 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:13,674 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:15,675 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:17,676 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:19,677 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:21,678 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:23,679 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:25,680 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:26,580 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:57:26,581 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:57:27,680 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:29,681 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:31,682 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:33,683 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:35,201 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:57:35,684 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:37,685 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:39,686 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:41,687 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:41,713 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:57:41,713 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:57:43,688 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:45,689 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:47,690 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:49,691 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:51,692 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:53,693 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:55,694 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:56,842 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:57:56,843 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:57:57,695 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:57:59,696 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:01,697 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:03,698 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:05,274 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:58:05,699 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:07,700 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:09,701 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:11,702 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:11,974 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:58:11,975 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:58:13,703 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:15,704 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:17,705 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:19,706 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:21,706 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:23,707 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:25,708 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:27,105 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:58:27,106 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:58:27,709 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:29,710 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:31,710 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:33,711 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:35,346 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:58:35,712 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:37,713 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:39,713 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:41,714 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:42,240 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:58:42,240 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:58:43,715 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:45,716 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:47,717 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:49,718 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:51,718 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:53,719 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:55,720 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:57,371 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:58:57,371 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:58:57,721 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:58:59,722 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:01,723 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:05,421 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:59:12,500 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:59:12,501 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:59:20,352 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 04:59:20,352 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 04:59:20,353 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 04:59:20,730 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 04:59:21,730 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:23,731 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:25,732 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:27,630 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:59:27,631 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:59:27,733 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:29,734 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:31,735 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:33,736 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:35,494 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 04:59:35,737 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:37,738 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:39,739 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:41,739 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:42,769 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:59:42,769 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:59:43,740 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:45,741 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:47,742 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:49,743 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:51,743 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:53,744 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:55,745 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:57,746 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 04:59:57,899 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 04:59:57,899 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 04:59:59,747 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:01,748 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:03,749 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:05,568 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:00:05,750 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:07,751 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:09,752 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:11,753 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:13,033 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:00:13,033 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:00:13,754 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:15,755 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:17,756 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:19,757 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:21,758 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:23,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:25,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:27,760 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:28,169 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:00:28,169 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:00:29,761 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:31,762 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:33,763 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:35,643 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:00:35,764 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:37,765 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:39,766 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:41,767 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:43,365 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:00:43,365 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:00:43,768 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:45,768 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:47,769 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:49,770 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:51,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:53,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:55,772 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:57,773 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:00:58,494 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:00:58,494 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:00:59,773 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:01,774 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:03,775 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:05,715 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:01:05,775 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:07,776 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:09,777 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:11,778 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:13,628 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:01:13,629 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:01:13,779 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:15,779 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:17,780 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:19,781 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:21,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:23,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:25,783 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:27,784 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:28,760 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:01:28,760 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:01:29,784 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:31,785 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:33,786 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:35,786 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:35,792 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:01:37,787 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:39,788 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:41,789 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:43,790 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:43,892 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:01:43,893 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:01:45,790 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:47,791 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:49,792 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:51,793 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:53,794 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:55,794 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:57,795 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:01:59,022 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:01:59,022 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:01:59,796 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:01,797 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:03,797 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:05,798 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:05,868 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:02:07,799 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:09,800 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:11,801 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:13,802 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:14,155 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:02:14,156 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:02:15,803 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:17,804 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:19,805 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:21,805 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:23,806 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:25,807 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:27,808 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:29,286 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:02:29,286 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:02:29,809 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:31,810 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:33,811 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:35,812 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:35,943 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:02:37,812 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:39,813 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:41,813 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:43,814 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:44,423 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:02:44,423 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:02:45,815 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:47,816 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:49,816 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:51,817 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:53,818 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:55,818 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:57,819 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:02:59,552 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:02:59,553 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:02:59,820 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:01,820 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:03,821 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:05,822 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:06,015 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:03:07,823 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:09,823 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:11,824 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:13,825 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:14,688 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:03:14,689 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:03:15,826 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:17,827 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:19,828 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:21,829 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:23,829 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:25,830 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:27,831 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:29,819 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:03:29,819 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:03:29,832 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:31,833 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:34,834 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:36,088 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:03:36,835 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:38,836 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:40,837 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:42,838 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:44,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:44,950 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:03:44,950 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:03:46,840 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:03:48,841 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:00,085 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:04:00,085 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:04:06,162 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:04:06,477 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:04:06,478 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:04:06,478 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:04:06,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:04:07,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:08,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:10,850 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:12,851 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:14,852 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:15,276 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:04:15,276 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:04:16,853 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:18,854 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:20,854 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:22,855 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:24,856 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:26,857 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:28,858 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:30,414 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:04:30,415 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:04:30,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:32,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:34,860 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:36,234 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:04:36,861 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:38,862 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:40,863 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:42,864 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:44,865 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:45,547 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:04:45,547 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:04:46,866 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:48,866 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:50,867 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:52,868 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:54,869 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:56,870 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:04:58,871 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:00,677 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:05:00,678 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:05:00,872 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:02,873 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:04,874 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:06,308 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:05:06,875 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:08,876 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:10,877 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:12,878 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:14,879 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:15,808 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:05:15,809 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:05:16,880 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:18,881 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:20,882 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:22,883 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:24,884 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:26,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:28,886 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:30,887 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:30,938 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:05:30,939 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:05:32,888 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:34,889 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:36,384 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:05:36,890 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:38,891 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:40,891 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:42,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:44,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:46,075 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:05:46,076 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:05:46,894 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:48,895 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:50,896 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:52,897 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:54,898 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:56,899 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:05:58,900 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:00,900 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:01,209 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:06:01,209 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:06:02,901 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:04,902 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:06,459 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:06:06,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:08,904 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:10,905 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:12,906 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:14,907 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:16,339 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:06:16,340 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:06:16,909 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:18,909 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:20,910 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:22,911 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:24,912 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:26,913 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:28,914 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:30,915 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:31,473 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:06:31,474 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:06:32,916 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:34,917 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:36,535 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:06:36,918 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:38,919 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:40,920 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:42,921 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:44,922 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:46,614 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:06:46,614 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:06:46,923 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:48,924 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:50,925 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:52,926 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:54,927 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:56,928 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:06:58,929 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:00,930 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:01,745 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:07:01,745 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:07:02,931 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:04,932 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:06,612 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:07:06,933 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:08,934 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:10,935 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:12,936 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:14,936 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:16,889 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:07:16,890 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:07:16,937 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:18,938 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:20,939 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:22,940 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:24,941 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:26,942 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:28,943 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:30,944 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:32,028 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:07:32,029 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:07:32,945 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:34,946 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:36,686 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:07:36,947 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:38,948 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:40,949 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:42,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:44,951 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:46,952 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:47,157 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:07:47,158 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:07:48,953 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:50,954 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:52,955 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:54,956 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:56,957 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:07:58,958 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:00,959 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:02,288 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:08:02,289 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:08:02,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:04,961 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:06,759 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:08:06,962 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:08,962 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:10,963 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:12,964 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:14,965 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:16,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:17,427 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:08:17,427 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:08:18,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:20,967 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:22,968 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:24,968 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:26,969 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:28,970 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:30,970 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:32,558 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:08:32,558 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:08:32,971 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:34,972 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:36,835 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:08:47,687 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:08:47,687 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:08:52,629 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:08:52,629 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:08:52,630 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:08:52,978 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:08:54,978 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:56,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:08:58,980 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:00,981 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:02,816 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:09:02,816 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:09:02,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:04,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:06,910 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:09:06,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:08,984 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:10,985 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:12,986 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:14,987 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:16,988 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:17,970 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:09:17,970 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:09:18,988 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:20,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:22,990 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:24,991 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:26,991 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:28,992 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:30,993 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:32,994 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:33,100 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:09:33,101 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:09:34,995 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:36,986 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:09:36,996 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:38,996 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:40,997 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:42,998 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:44,999 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:47,000 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:48,233 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:09:48,233 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:09:49,001 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:51,002 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:53,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:55,004 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:57,005 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:09:59,006 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:01,006 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:03,007 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:03,373 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:10:03,374 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:10:05,008 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:07,009 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:07,063 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:10:09,010 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:11,011 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:13,012 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:15,013 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:17,014 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:18,515 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:10:18,515 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:10:19,014 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:21,015 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:23,016 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:25,017 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:27,018 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:29,019 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:31,020 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:33,021 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:33,647 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:10:33,647 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:10:35,022 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:37,022 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:37,138 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:10:39,023 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:41,024 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:43,025 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:45,026 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:47,027 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:48,783 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:10:48,784 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:10:49,027 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:51,028 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:53,029 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:55,030 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:57,031 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:10:59,032 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:01,033 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:03,034 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:03,916 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:11:03,917 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:11:05,035 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:07,036 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:07,212 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:11:09,037 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:11,038 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:13,038 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:15,039 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:17,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:19,041 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:19,047 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:11:19,048 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:11:21,042 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:23,043 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:25,044 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:27,045 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:29,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:31,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:33,047 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:34,176 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:11:34,177 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:11:35,048 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:37,049 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:37,284 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:11:39,050 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:41,050 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:43,051 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:45,052 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:47,052 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:49,053 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:49,308 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:11:49,308 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:11:51,054 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:53,055 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:55,055 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:57,056 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:11:59,057 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:01,058 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:03,058 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:04,450 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:12:04,450 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:12:05,059 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:07,060 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:07,362 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:12:09,060 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:11,061 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:13,062 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:16,063 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:18,064 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:19,584 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:12:19,584 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:12:20,065 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:22,066 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:24,066 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:26,067 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:28,068 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:30,069 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:32,069 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:34,070 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:34,713 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:12:34,714 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:12:36,071 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:37,437 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:12:38,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:40,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:42,073 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:44,074 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:46,075 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:48,076 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:49,844 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:12:49,844 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:12:50,076 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:52,077 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:54,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:56,079 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:12:58,079 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:00,080 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:02,081 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:04,082 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:04,975 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:13:04,976 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:13:06,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:07,514 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:13:08,084 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:10,085 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:12,086 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:14,086 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:16,087 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:18,088 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:20,089 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:20,107 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:13:20,108 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:13:22,090 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:35,239 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:13:35,239 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:13:37,588 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:13:38,785 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:13:38,785 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:13:38,786 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:13:39,096 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:13:40,097 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:41,097 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:42,098 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:44,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:46,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:48,100 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:50,101 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:50,425 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:13:50,425 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:13:52,102 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:54,102 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:56,103 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:13:58,104 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:00,105 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:02,106 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:04,107 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:05,555 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:14:05,556 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:14:06,107 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:07,665 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:14:08,108 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:10,109 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:12,110 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:14,111 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:16,112 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:18,113 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:20,113 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:20,687 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:14:20,688 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:14:22,114 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:24,115 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:26,116 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:28,117 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:30,118 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:32,119 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:34,119 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:35,818 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:14:35,818 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:14:36,120 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:37,739 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:14:38,121 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:40,122 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:42,123 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:44,123 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:46,124 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:48,125 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:50,126 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:50,951 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:14:50,951 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:14:52,127 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:54,128 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:56,128 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:14:58,129 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:00,130 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:02,131 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:04,131 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:06,081 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:15:06,081 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:15:06,132 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:07,813 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:15:08,133 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:10,134 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:12,134 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:14,135 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:16,136 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:18,137 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:20,138 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:21,215 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:15:21,216 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:15:22,138 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:24,139 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:26,140 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:28,141 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:30,141 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:32,142 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:34,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:36,144 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:36,347 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:15:36,347 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:15:37,888 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:15:38,144 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:40,145 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:42,146 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:44,147 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:46,147 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:48,148 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:50,149 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:51,479 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:15:51,479 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:15:52,150 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:54,150 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:56,151 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:15:58,152 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:00,153 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:02,153 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:04,154 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:06,155 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:06,609 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:16:06,609 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:16:07,962 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:16:08,156 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:10,156 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:12,157 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:14,158 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:16,159 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:18,159 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:20,160 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:21,742 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:16:21,743 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:16:22,161 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:24,162 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:26,163 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:28,163 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:30,164 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:32,165 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:34,166 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:36,167 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:36,873 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:16:36,873 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:16:38,042 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:16:38,168 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:40,168 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:42,169 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:44,170 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:46,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:48,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:50,172 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:52,004 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:16:52,005 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:16:52,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:54,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:56,174 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:16:58,175 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:00,176 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:02,176 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:04,177 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:06,178 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:07,135 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:17:07,135 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:17:08,112 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:17:08,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:10,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:12,180 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:14,181 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:16,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:18,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:20,183 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:22,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:22,271 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:17:22,271 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:17:24,185 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:26,185 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:28,186 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:30,187 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:32,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:34,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:36,189 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:37,402 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:17:37,402 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:17:38,186 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:17:38,190 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:40,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:42,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:44,192 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:46,193 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:48,194 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:50,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:52,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:52,538 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:17:52,538 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:17:54,196 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:56,197 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:17:58,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:00,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:02,199 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:04,200 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:06,201 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:07,669 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:18:07,670 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:18:08,202 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:08,260 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:18:22,800 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:18:22,800 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:18:24,928 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:18:24,928 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:18:24,929 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:18:25,208 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:18:26,208 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:28,209 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:30,210 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:32,210 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:34,211 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:36,212 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:37,929 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:18:37,929 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:18:38,213 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:38,329 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:18:40,213 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:42,214 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:44,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:46,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:48,216 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:50,217 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:52,218 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:53,127 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:18:53,128 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:18:54,218 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:56,219 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:18:58,220 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:00,220 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:03,221 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:05,222 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:07,223 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:08,261 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:19:08,262 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:19:08,402 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:19:09,223 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:11,224 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:13,225 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:15,225 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:17,226 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:19,227 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:21,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:23,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:23,391 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:19:23,391 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:19:25,229 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:27,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:29,231 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:31,231 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:33,232 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:35,233 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:37,233 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:38,474 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:19:38,523 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:19:38,524 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:19:39,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:41,235 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:43,236 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:45,236 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:47,237 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:49,238 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:51,239 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:53,240 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:53,654 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:19:53,654 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:19:55,240 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:57,241 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:19:59,242 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:01,242 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:03,243 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:05,244 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:07,244 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:08,548 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:20:08,787 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:20:08,787 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:20:09,245 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:11,245 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:13,247 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:15,247 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:17,248 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:19,249 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:21,249 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:23,250 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:23,917 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:20:23,917 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:20:25,251 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:27,251 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:29,252 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:31,253 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:33,253 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:35,254 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:37,255 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:38,622 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:20:39,049 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:20:39,050 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:20:39,255 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:41,256 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:43,257 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:45,257 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:47,258 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:49,259 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:51,259 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:53,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:54,197 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:20:54,198 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:20:55,261 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:57,261 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:20:59,262 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:01,263 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:03,264 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:05,264 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:07,265 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:08,694 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:21:09,266 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:09,327 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:21:09,327 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:21:11,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:13,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:15,268 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:17,269 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:19,269 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:21,270 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:23,271 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:24,460 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:21:24,460 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:21:25,271 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:27,272 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:29,273 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:31,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:33,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:35,275 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:37,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:38,768 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:21:39,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:39,593 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:21:39,594 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:21:41,277 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:43,278 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:45,279 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:47,280 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:49,280 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:51,281 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:53,282 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:54,725 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:21:54,725 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:21:55,282 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:57,283 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:21:59,284 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:01,285 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:03,285 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:05,286 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:07,287 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:08,841 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:22:09,288 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:09,855 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:22:09,855 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:22:11,288 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:13,289 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:15,290 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:17,291 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:19,291 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:21,292 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:23,293 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:24,989 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:22:24,989 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:22:25,294 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:27,295 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:29,296 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:31,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:33,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:35,298 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:37,299 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:38,915 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:22:39,300 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:40,121 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:22:40,121 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:22:41,301 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:43,301 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:45,302 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:47,303 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:49,304 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:51,304 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:53,305 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:22:55,254 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:22:55,255 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:23:08,988 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:23:10,385 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:23:10,385 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:23:11,062 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:23:11,063 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:23:11,063 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:23:11,312 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:23:12,313 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:13,313 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:15,314 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:17,315 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:19,316 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:21,316 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:23,317 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:25,318 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:25,558 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:23:25,558 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:23:27,319 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:29,320 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:31,321 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:33,322 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:35,322 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:37,323 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:39,062 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:23:39,324 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:40,692 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:23:40,693 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:23:41,325 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:43,326 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:45,326 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:47,327 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:49,328 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:51,329 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:53,330 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:55,331 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:55,826 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:23:55,826 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:23:57,332 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:23:59,332 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:01,333 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:03,334 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:05,335 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:07,336 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:09,138 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:24:09,337 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:10,956 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:24:10,957 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:24:11,338 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:13,338 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:15,339 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:17,340 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:19,341 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:21,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:23,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:25,343 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:26,088 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:24:26,089 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:24:27,344 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:29,345 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:31,346 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:33,347 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:35,347 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:37,348 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:39,211 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:24:39,349 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:41,226 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:24:41,226 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:24:41,350 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:43,351 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:45,352 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:47,352 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:49,353 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:51,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:53,355 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:55,355 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:56,355 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:24:56,355 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:24:57,357 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:24:59,358 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:01,359 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:03,360 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:05,361 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:07,362 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:09,285 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:25:09,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:11,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:11,487 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:25:11,487 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:25:13,364 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:15,365 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:17,366 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:19,367 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:21,368 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:23,369 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:25,370 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:26,620 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:25:26,620 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:25:27,371 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:29,371 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:31,372 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:33,373 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:35,374 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:37,375 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:39,363 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:25:39,376 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:41,377 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:41,749 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:25:41,749 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:25:43,378 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:45,379 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:47,380 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:49,381 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:51,382 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:53,383 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:55,384 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:56,881 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:25:56,881 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:25:57,385 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:25:59,385 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:01,386 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:03,387 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:05,388 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:07,389 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:09,390 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:09,437 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:26:11,391 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:12,011 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:26:12,011 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:26:13,392 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:15,393 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:17,394 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:19,395 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:21,395 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:23,396 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:25,397 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:27,142 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:26:27,142 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:26:27,398 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:29,399 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:31,400 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:33,401 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:35,402 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:37,403 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:39,404 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:39,513 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:26:41,405 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:42,303 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:26:42,303 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:26:43,406 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:45,407 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:47,408 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:49,409 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:51,410 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:53,411 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:55,412 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:57,413 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:26:57,434 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:26:57,434 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:26:59,414 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:01,415 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:03,415 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:05,416 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:07,417 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:09,418 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:09,587 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:27:11,419 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:12,562 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:27:12,562 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:27:14,421 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:16,422 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:18,423 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:20,423 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:22,424 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:24,425 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:26,426 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:27,692 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:27:27,692 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:27:28,427 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:30,428 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:32,429 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:34,430 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:36,431 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:38,432 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:39,664 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:27:40,433 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:42,823 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:27:42,824 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:27:57,207 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:27:57,208 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:27:57,208 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:27:57,439 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:27:57,958 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:27:57,958 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:27:58,440 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:27:59,440 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:00,441 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:02,442 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:04,442 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:06,443 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:08,444 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:09,738 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:28:10,445 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:12,446 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:13,133 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:28:13,133 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:28:14,446 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:16,447 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:18,448 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:20,449 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:22,450 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:24,451 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:26,452 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:28,264 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:28:28,265 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:28:28,453 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:30,454 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:32,455 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:34,455 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:36,456 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:38,457 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:39,810 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:28:40,457 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:42,458 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:43,400 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:28:43,400 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:28:44,459 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:46,460 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:48,461 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:50,461 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:52,462 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:54,463 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:56,464 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:58,464 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:28:58,529 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:28:58,530 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:29:00,465 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:02,466 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:04,467 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:06,468 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:08,469 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:09,883 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:29:10,470 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:12,471 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:13,659 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:29:13,659 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:29:14,471 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:16,472 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:18,473 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:20,474 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:22,475 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:24,475 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:26,476 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:28,477 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:28,808 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:29:28,808 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:29:30,478 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:32,478 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:34,479 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:36,480 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:38,481 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:39,956 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:29:40,482 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:42,482 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:43,939 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:29:43,940 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:29:44,483 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:46,484 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:48,485 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:50,486 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:52,486 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:54,487 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:56,488 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:58,489 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:29:59,072 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:29:59,073 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:30:00,490 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:02,491 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:04,491 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:06,492 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:08,493 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:10,032 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:30:10,494 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:12,495 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:14,203 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:30:14,203 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:30:14,495 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:16,496 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:18,497 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:20,498 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:22,499 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:24,500 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:26,500 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:28,501 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:29,338 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:30:29,339 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:30:30,502 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:32,503 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:34,504 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:36,504 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:38,505 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:40,109 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:30:40,506 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:42,507 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:44,472 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:30:44,472 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:30:44,507 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:46,508 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:48,509 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:50,510 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:52,511 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:54,512 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:56,513 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:58,514 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:30:59,604 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:30:59,604 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:31:00,514 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:02,515 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:04,516 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:06,516 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:08,517 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:10,181 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:31:10,518 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:12,519 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:14,519 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:14,741 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:31:14,741 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:31:16,520 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:18,521 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:20,522 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:22,523 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:24,524 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:26,524 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:28,525 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:29,875 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:31:29,875 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:31:30,526 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:32,527 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:34,528 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:36,529 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:38,529 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:40,251 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:31:40,530 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:42,531 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:44,532 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:45,007 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:31:45,007 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:31:46,532 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:48,533 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:50,534 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:52,535 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:54,535 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:56,536 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:31:58,537 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:00,141 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:32:00,141 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:32:00,538 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:02,539 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:04,540 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:06,540 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:08,541 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:10,325 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:32:10,542 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:12,542 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:14,543 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:15,277 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:32:15,277 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:32:16,544 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:18,544 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:20,545 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:22,546 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:24,547 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:26,548 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:30,410 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:32:30,410 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:32:40,394 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:32:43,355 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:32:43,356 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:32:43,359 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:32:43,554 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:32:44,555 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:45,544 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:32:45,544 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:32:45,555 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:46,555 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:48,556 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:50,557 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:52,558 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:54,559 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:56,559 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:32:58,560 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:00,561 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:00,725 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:33:00,725 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:33:02,562 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:04,563 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:06,563 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:08,564 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:10,464 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:33:10,565 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:12,566 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:14,566 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:15,860 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:33:15,860 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:33:16,567 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:19,568 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:21,569 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:23,569 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:25,570 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:27,571 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:29,572 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:30,991 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:33:30,991 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:33:31,572 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:33,573 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:35,574 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:37,575 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:39,576 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:40,537 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:33:41,577 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:43,577 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:45,578 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:46,121 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:33:46,121 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:33:47,579 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:49,580 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:51,581 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:53,582 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:55,582 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:57,583 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:33:59,584 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:01,274 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:34:01,275 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:34:01,585 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:03,586 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:05,586 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:07,587 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:09,588 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:10,608 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:34:11,589 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:13,590 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:15,590 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:16,404 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:34:16,405 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:34:17,591 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:19,592 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:21,593 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:23,593 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:25,594 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:27,595 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:29,595 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:31,536 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:34:31,536 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:34:31,596 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:33,597 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:35,598 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:37,598 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:39,599 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:40,680 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:34:41,600 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:43,600 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:45,601 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:46,668 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:34:46,668 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:34:47,602 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:49,603 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:51,604 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:53,604 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:55,605 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:57,606 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:34:59,606 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:01,607 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:01,798 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:35:01,798 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:35:03,608 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:05,609 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:07,609 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:09,610 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:10,753 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:35:11,611 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:13,611 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:15,612 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:16,928 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:35:16,929 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:35:17,613 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:19,614 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:21,615 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:23,615 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:25,616 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:27,617 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:29,618 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:31,618 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:32,079 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:35:32,080 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:35:33,619 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:35,620 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:37,621 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:39,621 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:40,822 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:35:41,622 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:43,623 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:45,624 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:47,209 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:35:47,209 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:35:47,625 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:49,625 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:51,626 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:53,627 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:55,628 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:57,629 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:35:59,630 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:01,630 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:02,340 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:36:02,341 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:36:03,631 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:05,632 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:07,633 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:09,633 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:10,885 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:36:11,634 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:13,635 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:15,635 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:17,472 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:36:17,473 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:36:17,636 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:19,637 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:21,637 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:23,638 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:25,639 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:27,640 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:29,641 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:31,641 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:32,603 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:36:32,603 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:36:33,642 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:35,643 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:37,644 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:39,645 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:40,950 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:36:41,645 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:43,646 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:45,647 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:47,648 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:47,733 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:36:47,733 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:36:49,648 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:51,649 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:53,650 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:55,650 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:57,651 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:36:59,652 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:01,653 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:02,864 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:37:02,864 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:37:03,654 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:05,654 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:07,655 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:09,656 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:11,014 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:37:11,657 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:17,994 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:37:17,994 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:37:29,479 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:37:29,479 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:37:29,480 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:37:29,664 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:37:30,664 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:31,664 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:33,191 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:37:33,192 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:37:33,665 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:35,666 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:37,667 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:39,668 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:41,081 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:37:41,668 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:43,669 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:45,670 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:47,671 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:48,330 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:37:48,331 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:37:49,672 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:51,673 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:53,673 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:55,674 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:57,675 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:37:59,676 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:01,677 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:03,471 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:38:03,472 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:38:03,678 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:05,678 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:07,679 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:09,680 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:11,147 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:38:11,681 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:13,681 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:15,682 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:17,683 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:18,606 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:38:18,606 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:38:19,684 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:21,685 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:23,685 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:25,686 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:27,687 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:29,688 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:31,688 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:33,689 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:33,805 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:38:33,805 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:38:35,690 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:37,691 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:39,692 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:41,214 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:38:41,692 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:43,693 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:45,694 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:47,695 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:48,935 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:38:48,935 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:38:49,695 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:51,696 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:53,697 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:55,698 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:57,698 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:38:59,699 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:01,700 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:03,700 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:04,068 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:39:04,068 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:39:05,701 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:07,702 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:09,703 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:11,280 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:39:11,704 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:13,704 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:15,705 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:17,706 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:19,205 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:39:19,206 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:39:19,707 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:21,707 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:23,708 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:25,709 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:27,710 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:29,710 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:31,711 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:33,712 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:34,343 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:39:34,344 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:39:35,713 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:37,713 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:39,714 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:41,348 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:39:41,715 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:43,716 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:45,717 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:47,717 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:49,474 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:39:49,474 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:39:49,718 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:51,719 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:53,720 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:55,721 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:57,722 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:39:59,722 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:01,723 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:03,725 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:04,613 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:40:04,614 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:40:06,726 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:08,727 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:10,728 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:11,416 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:40:12,729 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:14,730 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:16,731 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:18,732 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:19,748 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:40:19,748 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:40:20,733 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:22,734 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:24,735 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:26,736 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:28,737 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:30,738 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:32,739 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:34,740 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:34,881 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:40:34,881 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:40:36,741 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:38,741 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:40,742 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:41,486 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:40:42,743 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:44,744 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:46,745 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:48,746 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:50,010 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:40:50,011 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:40:50,747 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:52,748 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:54,748 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:56,749 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:40:58,750 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:00,751 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:02,752 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:04,753 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:05,146 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:41:05,146 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:41:06,753 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:08,754 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:10,755 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:11,553 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:41:12,756 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:14,756 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:16,757 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:18,758 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:20,281 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:41:20,282 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:41:20,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:22,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:24,760 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:26,761 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:28,761 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:30,762 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:32,763 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:34,764 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:35,413 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:41:35,414 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:41:36,764 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:38,765 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:40,766 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:41,626 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:41:42,767 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:44,768 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:46,768 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:48,769 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:50,548 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:41:50,549 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:41:50,770 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:52,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:54,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:56,772 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:41:58,773 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:05,678 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:42:05,678 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:42:11,697 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:42:15,603 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:42:15,604 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:42:15,604 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:42:15,779 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:42:16,780 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:18,781 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:20,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:20,809 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:42:20,809 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:42:22,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:24,783 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:26,784 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:28,785 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:30,785 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:32,786 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:34,787 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:35,960 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:42:35,961 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:42:36,788 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:38,789 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:40,790 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:41,772 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:42:42,790 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:44,791 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:46,792 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:48,793 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:50,793 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:51,092 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:42:51,093 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:42:52,794 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:54,795 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:56,796 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:42:58,797 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:00,797 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:02,798 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:04,799 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:06,224 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:43:06,224 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:43:06,800 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:08,801 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:10,802 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:11,849 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:43:12,803 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:14,804 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:16,805 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:18,806 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:20,807 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:21,357 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:43:21,357 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:43:22,808 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:24,809 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:26,810 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:28,811 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:30,812 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:32,813 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:34,814 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:36,487 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:43:36,487 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:43:36,815 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:38,816 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:40,817 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:41,922 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:43:42,818 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:44,819 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:46,820 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:48,821 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:50,822 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:51,618 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:43:51,619 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:43:52,822 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:54,823 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:56,824 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:43:58,825 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:00,826 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:02,827 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:04,827 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:06,753 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:44:06,753 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:44:06,828 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:08,829 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:10,830 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:11,997 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:44:12,831 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:14,831 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:16,832 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:18,833 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:20,834 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:21,884 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:44:21,884 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:44:22,835 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:24,836 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:26,837 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:28,838 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:30,838 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:32,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:34,840 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:36,841 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:37,016 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:44:37,016 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:44:38,842 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:40,843 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:42,072 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:44:42,843 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:44,844 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:46,845 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:48,847 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:50,848 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:52,147 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:44:52,147 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:44:52,848 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:54,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:56,850 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:44:58,851 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:00,852 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:02,853 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:04,853 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:06,854 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:07,279 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:45:07,280 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:45:08,855 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:10,856 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:12,145 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:45:12,857 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:14,858 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:16,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:18,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:20,860 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:22,410 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:45:22,411 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:45:22,861 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:24,862 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:26,863 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:28,864 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:30,865 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:32,866 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:34,867 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:36,868 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:37,545 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:45:37,546 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:45:38,868 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:40,869 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:42,222 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:45:42,870 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:44,871 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:46,872 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:48,873 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:50,874 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:52,676 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:45:52,677 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:45:52,875 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:54,876 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:56,876 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:45:58,877 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:00,878 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:02,878 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:04,879 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:06,880 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:07,807 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:46:07,807 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:46:08,881 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:10,882 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:12,295 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:46:12,883 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:14,884 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:16,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:18,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:20,886 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:22,887 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:22,940 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:46:22,940 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:46:24,888 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:26,889 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:28,890 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:30,890 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:32,891 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:34,892 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:36,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:38,070 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:46:38,070 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:46:38,894 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:40,895 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:42,367 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:46:42,895 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:44,896 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:46:53,201 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:46:53,202 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:47:01,742 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:47:01,742 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:47:01,743 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:47:01,902 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:47:02,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:03,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:04,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:06,904 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:08,376 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:47:08,377 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:47:08,905 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:10,906 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:12,440 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:47:12,907 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:14,908 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:16,908 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:18,909 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:20,910 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:22,911 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:23,509 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:47:23,509 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:47:24,912 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:26,913 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:28,914 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:30,914 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:32,915 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:34,916 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:37,917 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:38,659 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:47:38,660 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:47:39,918 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:41,919 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:42,514 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:47:43,920 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:45,921 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:47,921 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:49,922 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:51,923 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:53,788 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:47:53,789 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:47:53,924 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:55,925 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:57,926 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:47:59,926 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:01,927 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:03,928 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:05,929 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:07,929 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:08,926 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:48:08,926 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:48:09,930 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:11,931 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:12,590 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:48:13,932 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:15,933 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:17,933 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:19,934 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:21,935 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:23,936 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:24,058 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:48:24,059 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:48:25,937 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:27,938 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:29,940 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:31,940 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:33,941 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:35,942 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:37,942 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:39,195 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:48:39,196 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:48:39,943 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:41,944 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:42,668 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:48:43,945 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:45,946 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:47,947 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:49,948 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:51,948 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:53,949 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:54,331 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:48:54,331 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:48:55,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:57,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:48:59,951 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:01,952 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:03,953 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:05,954 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:07,954 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:09,471 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:49:09,472 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:49:09,955 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:11,956 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:12,741 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:49:13,957 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:15,957 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:17,958 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:19,959 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:21,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:23,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:24,607 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:49:24,607 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:49:25,961 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:27,962 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:29,963 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:31,964 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:33,964 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:35,965 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:37,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:39,737 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:49:39,738 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:49:39,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:41,967 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:42,815 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:49:43,968 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:45,968 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:47,969 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:49,970 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:51,970 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:53,971 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:54,868 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:49:54,869 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:49:55,972 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:57,973 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:49:59,974 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:01,974 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:03,975 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:05,976 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:07,976 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:09,977 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:10,001 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:50:10,001 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:50:11,978 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:12,888 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:50:13,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:15,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:17,980 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:19,981 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:21,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:23,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:25,134 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:50:25,134 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:50:25,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:27,984 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:29,984 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:31,985 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:33,986 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:35,986 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:37,987 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:39,988 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:40,266 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:50:40,267 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:50:41,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:42,964 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:50:43,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:45,990 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:47,991 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:49,991 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:51,992 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:53,993 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:55,396 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:50:55,397 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:50:55,993 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:57,994 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:50:59,995 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:01,995 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:03,996 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:05,996 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:07,997 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:09,998 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:10,528 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:51:10,528 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:51:11,998 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:13,040 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:51:13,999 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:16,000 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:18,000 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:20,001 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:22,002 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:24,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:25,658 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:51:25,658 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:51:26,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:28,004 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:30,004 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:40,791 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:51:40,791 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:51:43,111 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:51:47,876 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:51:47,876 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:51:47,876 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:51:48,009 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:51:50,010 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:52,011 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:54,011 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:55,924 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:51:55,925 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:51:56,012 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:51:58,013 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:00,014 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:02,014 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:04,015 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:06,016 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:08,016 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:10,017 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:11,088 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:52:11,089 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:52:12,018 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:13,185 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:52:14,018 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:16,019 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:18,020 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:20,021 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:22,022 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:24,022 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:26,023 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:26,218 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:52:26,219 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:52:28,024 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:30,025 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:32,026 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:34,026 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:36,027 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:38,028 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:40,028 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:41,351 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:52:41,351 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:52:42,029 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:43,260 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:52:44,030 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:46,031 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:48,032 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:50,032 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:52,033 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:54,034 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:56,035 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:52:56,484 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:52:56,485 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:52:58,036 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:00,036 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:02,037 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:04,038 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:06,039 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:08,039 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:10,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:11,615 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:53:11,616 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:53:12,041 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:13,334 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:53:14,041 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:16,042 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:18,043 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:20,043 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:22,044 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:24,045 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:26,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:26,746 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:53:26,747 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:53:28,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:30,047 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:32,048 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:34,049 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:36,049 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:38,050 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:40,051 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:41,878 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:53:41,878 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:53:42,051 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:43,410 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:53:44,052 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:46,053 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:48,053 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:50,054 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:52,055 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:54,055 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:56,056 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:53:57,009 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:53:57,010 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:53:58,057 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:00,058 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:02,059 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:04,059 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:06,061 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:08,061 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:10,062 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:12,063 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:12,141 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:54:12,141 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:54:13,488 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:54:14,064 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:16,065 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:18,066 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:20,067 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:22,068 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:24,069 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:26,070 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:27,275 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:54:27,275 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:54:28,071 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:30,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:32,073 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:35,074 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:37,075 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:39,076 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:41,076 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:42,406 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:54:42,407 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:54:43,077 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:43,563 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:54:45,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:47,079 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:49,080 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:51,081 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:53,081 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:55,082 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:57,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:54:57,558 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:54:57,559 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:54:59,084 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:01,085 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:03,086 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:05,087 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:07,087 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:09,088 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:11,089 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:12,691 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:55:12,692 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:55:13,090 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:13,638 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:55:15,090 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:17,091 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:19,092 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:21,093 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:23,094 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:25,094 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:27,095 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:27,821 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:55:27,822 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:55:29,096 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:31,097 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:33,098 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:35,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:37,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:39,100 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:41,101 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:42,952 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:55:42,953 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:55:43,101 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:43,712 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:55:45,102 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:47,103 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:49,104 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:51,104 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:53,105 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:55,106 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:57,106 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:55:58,084 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:55:58,084 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:55:59,107 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:01,108 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:03,109 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:05,109 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:07,110 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:09,111 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:11,112 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:13,112 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:13,213 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:56:13,213 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:56:13,788 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:56:15,113 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:17,114 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:28,344 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:56:28,345 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:56:34,006 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 05:56:34,006 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 05:56:34,011 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 05:56:34,120 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 05:56:35,120 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:37,121 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:39,122 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:41,122 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:43,123 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:43,476 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:56:43,477 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:56:43,862 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:56:45,124 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:47,124 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:49,125 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:51,126 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:53,127 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:55,127 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:57,128 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:56:58,610 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:56:58,611 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:56:59,129 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:01,130 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:03,130 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:05,131 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:07,132 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:09,133 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:11,134 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:13,135 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:13,773 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:57:13,773 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:57:13,938 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:57:15,136 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:17,137 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:19,137 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:21,138 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:23,139 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:25,140 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:27,140 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:28,914 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:57:28,914 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:57:29,141 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:31,142 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:33,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:35,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:37,144 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:39,145 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:41,146 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:43,147 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:44,013 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:57:44,094 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:57:44,094 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:57:45,147 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:47,148 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:49,149 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:51,149 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:53,150 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:55,151 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:57,152 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:59,153 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:57:59,237 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:57:59,237 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:58:01,154 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:03,155 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:05,155 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:07,156 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:09,157 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:11,158 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:13,158 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:14,092 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:58:14,509 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:58:14,510 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:58:15,159 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:17,160 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:19,161 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:21,161 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:23,162 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:25,163 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:27,164 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:29,165 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:29,641 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:58:29,641 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:58:31,165 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:33,166 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:35,167 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:37,167 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:39,168 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:41,169 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:43,169 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:44,167 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:58:44,770 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:58:44,771 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:58:45,170 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:47,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:49,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:51,172 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:53,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:55,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:57,174 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:59,175 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:58:59,899 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:58:59,899 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:59:01,175 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:03,176 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:05,177 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:07,177 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:09,178 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:11,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:13,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:14,240 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:59:15,028 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:59:15,029 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:59:15,180 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:17,181 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:19,181 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:21,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:23,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:25,183 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:27,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:29,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:30,157 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:59:30,157 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:59:31,185 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:33,186 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:35,186 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:37,187 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:39,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:41,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:43,189 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:44,314 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 05:59:45,190 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:45,286 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 05:59:45,286 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 05:59:47,190 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:49,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:51,192 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:53,192 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:55,193 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:57,194 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 05:59:59,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:00,415 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:00:00,415 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:00:01,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:03,196 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:05,197 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:07,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:09,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:11,199 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:13,200 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:14,387 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:00:15,201 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:15,545 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:00:15,545 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:00:17,202 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:19,202 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:21,203 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:23,204 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:25,204 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:27,205 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:29,205 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:30,673 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:00:30,674 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:00:31,206 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:33,207 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:35,207 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:37,208 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:39,209 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:41,209 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:43,210 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:44,460 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:00:45,211 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:45,804 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:00:45,805 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:00:47,211 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:49,212 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:51,213 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:53,213 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:55,214 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:57,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:00:59,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:00,936 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:01:00,936 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:01:01,216 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:03,217 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:14,532 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:01:16,065 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:01:16,066 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:01:20,169 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:01:20,170 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:01:20,170 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:01:20,222 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:01:21,222 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:23,223 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:25,224 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:27,224 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:29,225 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:31,226 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:31,306 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:01:31,306 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:01:33,227 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:35,227 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:37,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:39,229 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:41,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:43,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:44,604 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:01:45,231 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:46,435 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:01:46,436 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:01:47,232 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:49,232 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:51,233 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:53,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:55,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:57,235 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:01:59,236 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:01,237 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:01,567 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:02:01,567 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:02:03,237 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:05,238 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:07,239 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:09,240 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:11,241 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:13,241 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:14,675 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:02:15,242 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:16,697 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:02:16,697 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:02:17,243 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:19,244 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:21,245 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:23,246 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:25,246 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:27,247 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:29,248 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:31,249 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:31,828 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:02:31,829 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:02:33,249 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:35,250 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:37,250 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:39,251 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:41,252 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:43,252 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:44,750 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:02:45,253 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:46,959 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:02:46,959 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:02:47,254 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:49,254 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:51,255 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:53,256 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:55,256 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:57,257 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:02:59,258 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:01,259 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:02,093 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:03:02,094 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:03:03,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:05,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:07,261 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:09,262 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:11,262 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:13,263 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:14,828 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:03:16,264 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:17,223 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:03:17,223 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:03:18,265 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:20,266 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:22,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:24,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:26,268 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:28,269 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:30,270 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:32,271 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:32,354 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:03:32,354 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:03:34,272 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:36,272 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:38,273 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:40,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:42,275 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:44,275 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:44,904 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:03:46,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:47,485 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:03:47,485 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:03:48,277 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:50,278 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:52,279 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:54,280 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:56,280 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:03:58,281 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:00,282 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:02,283 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:02,623 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:04:02,623 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:04:04,283 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:06,284 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:08,285 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:10,286 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:12,287 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:14,287 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:14,982 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:04:16,288 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:17,761 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:04:17,761 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:04:18,289 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:20,289 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:22,290 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:24,291 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:26,292 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:28,292 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:30,293 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:32,294 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:32,891 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:04:32,891 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:04:34,295 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:36,295 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:38,296 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:40,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:42,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:44,298 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:45,058 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:04:46,299 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:48,024 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:04:48,024 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:04:48,300 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:50,300 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:52,301 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:54,302 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:56,302 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:04:58,303 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:00,304 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:02,305 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:03,154 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:05:03,154 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:05:04,305 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:06,306 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:08,307 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:10,307 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:12,308 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:14,309 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:15,133 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:05:16,310 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:18,292 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:05:18,292 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:05:18,310 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:20,311 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:22,312 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:24,312 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:26,313 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:28,314 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:30,315 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:32,315 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:33,428 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:05:33,429 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:05:34,316 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:36,317 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:38,317 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:40,318 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:42,319 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:44,320 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:45,211 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:05:46,320 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:48,321 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:05:48,568 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:05:48,568 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:06:03,699 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:06:03,699 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:06:06,325 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:06:06,325 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:06:06,325 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:06:06,327 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:06:07,327 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:08,328 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:10,328 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:12,329 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:14,330 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:15,289 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:06:16,331 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:18,331 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:18,830 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:06:18,831 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:06:20,332 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:22,333 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:24,334 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:26,335 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:28,336 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:30,337 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:32,337 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:33,972 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:06:33,972 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:06:34,338 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:36,339 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:38,340 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:40,341 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:42,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:44,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:45,367 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:06:46,343 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:48,344 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:49,106 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:06:49,106 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:06:50,345 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:52,346 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:54,347 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:56,347 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:06:58,348 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:00,349 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:02,350 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:04,234 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:07:04,234 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:07:04,351 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:06,351 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:08,352 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:10,353 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:12,353 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:14,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:15,445 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:07:16,355 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:18,356 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:19,364 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:07:19,364 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:07:20,357 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:22,358 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:24,358 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:26,359 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:28,360 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:30,361 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:32,361 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:34,362 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:34,496 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:07:34,497 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:07:36,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:38,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:40,364 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:42,365 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:44,365 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:45,522 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:07:46,366 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:48,367 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:49,628 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:07:49,628 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:07:50,367 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:52,368 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:54,369 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:56,370 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:07:58,371 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:00,372 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:02,373 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:04,373 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:04,759 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:08:04,759 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:08:06,375 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:08,375 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:10,376 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:12,377 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:14,378 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:15,599 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:08:16,378 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:18,379 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:19,897 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:08:19,898 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:08:20,380 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:22,380 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:24,381 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:26,382 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:28,383 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:30,383 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:32,384 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:34,385 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:35,027 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:08:35,028 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:08:36,385 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:38,386 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:40,387 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:42,388 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:44,388 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:45,673 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:08:46,389 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:48,390 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:50,158 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:08:50,158 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:08:50,391 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:52,391 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:54,392 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:56,393 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:08:58,394 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:00,394 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:02,395 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:04,396 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:05,288 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:09:05,288 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:09:06,397 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:08,397 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:10,398 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:12,399 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:14,400 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:15,749 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:09:16,400 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:18,401 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:20,402 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:20,419 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:09:20,420 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:09:22,403 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:24,404 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:26,405 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:28,406 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:30,406 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:32,407 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:34,408 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:35,549 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:09:35,549 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:09:36,409 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:38,409 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:40,410 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:42,411 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:44,412 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:45,827 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:09:46,412 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:48,413 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:50,414 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:50,681 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:09:50,681 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:09:52,415 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:54,415 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:56,416 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:09:58,417 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:00,418 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:02,418 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:04,419 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:05,812 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:10:05,813 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:10:06,420 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:08,421 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:10,422 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:12,423 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:14,424 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:15,901 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:10:16,425 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:18,426 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:20,426 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:20,944 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:10:20,944 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:10:22,427 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:24,428 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:26,429 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:28,429 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:30,430 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:32,431 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:34,432 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:36,075 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:10:36,075 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:10:45,972 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:10:51,205 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:10:51,205 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:10:52,496 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:10:52,496 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:10:52,496 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:10:53,439 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:10:54,439 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:55,440 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:57,441 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:10:59,441 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:01,442 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:03,443 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:05,443 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:06,334 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:11:06,334 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:11:07,444 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:09,445 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:11,446 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:13,447 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:15,448 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:16,047 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:11:17,449 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:19,450 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:21,450 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:21,465 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:11:21,465 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:11:23,451 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:25,452 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:27,453 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:29,453 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:31,454 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:33,455 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:35,456 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:36,611 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:11:36,612 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:11:37,456 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:39,457 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:41,458 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:43,459 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:45,460 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:46,121 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:11:47,460 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:49,461 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:51,462 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:51,746 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:11:51,746 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:11:53,463 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:55,464 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:57,465 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:11:59,465 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:01,466 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:03,467 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:05,468 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:06,941 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:12:06,942 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:12:07,468 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:09,469 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:11,470 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:13,471 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:15,471 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:16,192 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:12:17,472 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:19,473 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:21,474 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:22,071 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:12:22,072 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:12:23,475 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:25,476 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:27,476 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:29,477 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:31,478 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:33,479 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:35,479 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:37,206 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:12:37,206 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:12:37,480 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:39,481 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:41,482 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:43,482 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:45,483 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:46,263 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:12:47,484 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:49,484 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:51,485 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:52,336 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:12:52,336 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:12:53,486 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:55,487 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:57,487 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:12:59,488 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:01,489 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:03,490 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:05,491 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:07,468 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:13:07,468 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:13:07,492 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:09,492 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:11,493 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:13,494 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:15,495 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:16,331 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:13:17,496 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:19,496 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:21,497 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:22,601 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:13:22,601 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:13:23,498 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:25,499 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:27,500 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:29,501 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:31,501 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:33,502 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:35,503 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:37,504 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:37,731 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:13:37,731 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:13:39,505 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:41,506 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:43,507 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:45,507 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:46,402 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:13:47,508 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:49,509 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:51,510 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:52,892 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:13:52,892 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:13:53,510 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:55,511 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:57,512 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:13:59,513 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:01,514 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:03,514 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:05,515 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:07,516 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:08,022 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:14:08,022 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:14:09,517 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:11,518 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:13,518 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:15,519 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:16,472 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:14:17,520 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:19,521 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:21,522 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:23,153 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:14:23,153 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:14:23,522 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:25,523 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:27,524 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:29,525 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:31,525 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:33,526 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:35,527 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:37,527 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:38,284 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:14:38,284 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:14:39,528 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:41,529 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:43,530 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:45,531 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:46,545 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:14:47,532 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:49,533 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:51,533 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:53,424 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:14:53,424 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:14:53,534 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:55,535 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:57,536 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:14:59,537 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:01,537 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:03,538 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:05,539 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:07,540 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:08,553 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:15:08,553 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:15:09,541 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:11,542 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:13,542 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:15,543 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:16,620 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:15:17,544 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:19,545 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:21,545 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:23,698 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:15:23,698 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:15:38,637 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:15:38,638 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:15:38,638 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:15:38,856 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:15:38,856 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:15:39,552 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:15:40,552 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:41,553 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:43,554 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:45,555 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:46,693 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:15:47,556 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:49,557 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:51,558 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:53,559 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:53,989 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:15:53,989 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:15:55,559 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:57,560 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:15:59,561 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:01,562 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:03,563 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:05,563 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:07,564 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:09,134 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:16:09,135 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:16:09,565 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:11,566 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:13,567 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:15,568 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:16,767 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:16:17,569 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:19,570 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:21,571 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:23,572 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:24,265 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:16:24,265 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:16:25,573 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:27,573 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:29,574 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:31,575 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:33,576 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:35,577 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:37,578 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:39,395 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:16:39,396 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:16:39,579 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:41,580 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:43,581 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:45,582 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:46,844 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:16:47,583 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:49,583 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:51,584 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:53,585 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:54,525 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:16:54,526 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:16:55,586 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:57,586 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:16:59,587 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:01,588 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:03,589 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:05,590 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:07,590 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:09,591 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:09,667 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:17:09,667 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:17:11,592 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:13,593 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:15,594 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:16,917 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:17:17,595 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:19,595 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:21,596 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:23,597 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:24,797 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:17:24,797 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:17:25,598 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:27,599 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:29,600 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:31,601 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:33,601 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:35,602 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:37,603 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:39,604 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:39,934 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:17:39,934 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:17:41,605 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:43,606 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:45,607 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:46,993 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:17:47,608 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:49,608 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:51,609 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:53,610 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:55,068 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:17:55,068 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:17:55,611 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:57,611 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:17:59,612 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:01,613 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:03,614 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:05,614 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:07,615 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:09,616 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:10,199 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:18:10,200 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:18:11,616 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:13,617 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:15,618 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:17,068 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:18:17,619 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:19,619 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:21,620 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:23,621 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:25,332 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:18:25,332 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:18:25,621 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:27,622 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:29,623 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:31,623 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:33,624 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:35,625 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:37,625 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:39,626 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:40,489 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:18:40,490 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:18:41,627 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:43,627 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:45,628 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:47,141 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:18:47,629 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:49,630 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:51,631 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:53,631 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:55,622 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:18:55,622 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:18:55,632 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:57,633 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:18:59,634 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:01,635 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:03,636 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:06,637 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:08,638 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:10,638 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:10,751 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:19:10,752 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:19:12,639 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:14,640 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:16,641 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:17,211 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:19:18,641 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:20,642 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:22,643 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:24,644 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:25,883 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:19:25,883 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:19:26,645 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:28,645 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:30,646 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:32,647 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:34,648 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:36,649 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:38,650 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:40,650 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:41,017 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:19:41,017 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:19:42,651 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:44,652 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:46,652 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:47,280 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:19:48,653 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:50,654 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:52,655 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:54,656 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:56,155 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:19:56,155 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:19:56,657 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:19:58,657 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:00,658 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:02,659 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:04,660 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:06,661 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:11,282 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:20:11,283 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:20:17,353 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:20:24,774 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:20:24,775 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:20:24,775 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:20:25,667 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:25,667 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:20:26,462 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:20:26,462 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:20:26,667 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:28,668 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:30,669 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:32,670 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:34,670 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:36,671 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:38,672 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:40,673 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:41,653 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:20:41,653 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:20:42,673 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:44,674 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:46,675 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:47,423 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:20:48,676 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:50,676 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:52,677 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:54,678 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:56,678 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:20:56,800 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:20:56,800 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:20:58,679 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:00,680 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:02,681 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:04,681 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:06,682 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:08,683 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:10,684 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:11,956 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:21:11,957 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:21:12,685 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:14,685 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:16,686 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:17,493 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:21:18,687 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:20,688 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:22,688 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:24,689 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:26,690 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:27,106 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:21:27,106 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:21:28,691 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:30,691 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:32,692 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:34,693 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:36,694 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:38,694 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:40,695 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:42,237 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:21:42,238 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:21:42,696 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:44,697 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:46,697 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:47,565 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:21:48,698 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:50,699 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:52,700 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:54,700 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:56,701 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:21:57,367 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:21:57,368 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:21:58,702 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:00,703 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:02,704 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:04,705 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:06,705 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:08,706 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:10,707 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:12,502 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:22:12,502 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:22:12,707 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:14,708 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:16,709 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:17,638 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:22:18,710 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:20,711 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:22,711 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:24,712 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:26,713 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:27,631 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:22:27,632 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:22:28,714 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:30,715 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:32,715 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:34,716 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:36,717 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:38,717 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:40,718 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:42,719 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:42,778 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:22:42,778 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:22:44,720 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:46,721 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:47,708 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:22:48,722 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:50,723 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:52,724 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:54,725 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:56,725 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:22:57,909 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:22:57,910 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:22:58,726 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:00,727 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:02,728 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:04,729 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:06,729 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:08,730 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:10,731 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:12,732 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:13,045 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:23:13,045 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:23:14,733 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:16,734 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:17,780 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:23:18,735 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:20,735 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:22,736 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:24,737 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:26,738 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:28,175 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:23:28,176 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:23:28,738 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:30,739 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:32,740 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:34,741 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:36,741 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:38,742 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:40,743 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:42,744 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:43,308 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:23:43,308 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:23:44,745 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:46,745 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:47,852 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:23:48,746 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:50,747 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:52,748 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:54,749 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:56,749 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:23:58,439 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:23:58,439 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:23:58,750 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:00,751 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:02,752 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:04,753 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:06,754 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:08,755 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:10,755 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:12,756 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:13,572 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:24:13,572 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:24:14,757 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:16,758 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:17,922 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:24:18,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:20,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:22,760 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:24,761 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:26,762 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:28,702 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:24:28,702 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:24:28,763 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:30,763 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:32,764 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:34,766 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:36,767 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:38,767 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:40,768 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:42,770 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:43,836 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:24:43,836 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:24:44,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:46,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:47,995 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:24:48,772 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:50,773 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:52,774 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:24:58,967 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:24:58,968 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:25:10,921 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:25:10,921 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:25:10,921 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:25:11,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:11,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:25:12,783 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:14,110 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:25:14,110 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:25:14,784 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:16,784 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:18,068 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:25:18,785 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:20,786 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:22,787 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:24,787 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:26,788 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:28,789 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:29,301 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:25:29,301 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:25:30,790 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:32,791 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:34,792 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:36,792 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:38,793 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:40,794 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:42,795 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:44,433 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:25:44,433 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:25:44,795 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:46,796 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:48,144 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:25:48,797 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:50,798 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:52,799 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:54,800 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:56,801 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:58,802 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:25:59,577 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:25:59,577 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:26:00,802 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:02,803 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:04,804 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:06,805 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:08,805 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:10,806 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:12,807 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:14,706 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:26:14,707 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:26:14,808 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:16,809 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:18,216 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:26:18,810 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:20,810 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:22,811 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:24,812 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:26,813 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:28,814 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:29,839 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:26:29,839 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:26:30,814 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:32,815 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:34,816 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:36,817 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:38,818 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:40,819 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:42,820 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:44,821 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:44,973 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:26:44,974 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:26:46,822 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:48,290 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:26:48,823 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:50,824 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:52,824 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:54,825 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:56,826 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:26:58,827 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:00,107 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:27:00,107 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:27:00,828 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:02,829 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:04,830 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:06,831 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:08,832 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:10,832 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:12,833 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:14,834 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:15,238 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:27:15,238 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:27:16,835 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:18,364 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:27:18,836 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:20,836 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:23,837 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:25,838 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:27,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:29,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:30,370 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:27:30,371 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:27:31,840 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:33,841 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:35,841 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:37,842 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:39,843 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:41,843 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:43,844 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:45,505 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:27:45,505 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:27:45,845 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:47,845 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:48,437 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:27:49,846 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:51,847 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:53,847 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:55,848 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:57,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:27:59,850 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:00,639 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:28:00,640 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:28:01,850 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:03,851 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:05,852 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:07,852 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:09,853 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:11,854 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:13,855 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:15,773 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:28:15,773 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:28:15,856 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:17,856 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:18,511 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:28:19,857 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:21,858 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:23,858 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:25,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:27,860 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:29,860 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:30,905 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:28:30,906 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:28:31,861 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:33,862 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:35,863 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:37,863 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:39,864 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:41,865 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:43,865 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:45,866 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:46,037 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:28:46,037 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:28:47,867 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:48,580 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:28:49,867 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:51,868 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:53,869 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:55,870 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:57,870 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:28:59,871 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:01,168 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:29:01,168 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:29:01,872 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:03,872 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:05,873 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:07,874 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:09,874 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:11,875 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:13,876 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:15,877 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:16,304 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:29:16,304 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:29:17,877 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:18,650 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:29:19,878 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:21,879 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:23,880 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:25,880 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:27,881 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:29,882 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:31,451 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:29:31,452 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:29:31,882 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:33,883 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:35,884 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:37,884 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:39,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:46,587 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:29:46,587 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:29:48,726 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:29:57,084 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:29:57,084 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:29:57,085 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:29:57,892 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:29:58,892 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:29:59,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:01,721 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:30:01,722 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:30:01,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:03,894 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:05,895 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:07,896 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:09,896 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:11,897 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:13,898 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:15,898 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:16,870 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:30:16,870 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:30:17,899 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:18,802 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:30:19,900 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:21,900 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:23,901 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:25,902 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:27,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:29,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:31,904 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:32,002 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:30:32,002 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:30:33,905 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:35,905 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:37,906 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:39,907 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:41,908 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:43,908 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:45,909 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:47,138 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:30:47,139 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:30:47,910 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:48,876 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:30:49,911 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:51,911 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:53,912 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:55,912 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:57,913 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:30:59,914 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:01,915 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:02,268 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:31:02,269 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:31:03,916 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:05,916 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:07,917 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:09,918 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:11,919 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:13,920 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:15,920 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:17,444 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:31:17,444 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:31:17,921 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:18,951 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:31:19,922 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:21,923 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:23,923 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:25,924 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:27,925 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:29,925 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:31,926 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:32,575 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:31:32,575 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:31:33,927 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:35,928 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:37,928 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:39,929 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:41,930 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:43,931 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:45,931 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:47,722 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:31:47,722 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:31:47,932 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:49,026 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:31:49,933 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:51,934 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:53,934 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:55,935 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:57,936 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:31:59,936 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:01,937 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:02,853 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:32:02,853 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:32:03,938 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:05,939 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:07,940 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:09,940 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:11,941 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:13,942 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:15,943 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:17,943 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:17,993 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:32:17,993 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:32:19,101 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:32:19,944 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:21,945 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:23,946 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:25,947 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:27,947 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:29,948 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:31,949 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:33,189 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:32:33,190 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:32:33,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:35,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:37,951 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:39,952 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:41,953 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:43,953 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:45,954 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:47,955 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:48,321 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:32:48,322 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:32:49,177 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:32:49,956 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:51,957 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:53,958 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:55,959 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:57,959 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:32:59,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:01,961 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:03,454 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:33:03,455 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:33:03,962 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:05,963 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:07,964 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:09,964 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:11,965 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:13,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:15,967 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:17,968 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:18,584 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:33:18,585 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:33:19,252 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:33:19,968 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:21,969 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:23,970 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:25,971 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:27,971 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:29,972 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:31,973 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:33,730 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:33:33,730 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:33:33,974 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:35,975 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:37,976 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:39,976 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:41,977 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:43,978 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:45,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:47,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:48,864 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:33:48,864 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:33:49,326 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:33:49,980 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:51,981 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:53,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:55,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:57,984 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:33:59,985 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:01,985 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:03,986 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:04,002 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:34:04,002 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:34:05,987 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:07,988 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:09,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:11,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:13,990 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:15,991 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:17,992 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:19,133 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:34:19,134 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:34:19,397 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:34:19,992 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:21,993 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:23,994 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:25,995 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:34,263 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:34:34,263 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:34:43,213 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:34:43,214 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:34:43,214 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:34:44,001 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:44,001 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:34:45,002 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:46,002 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:48,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:49,402 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:34:49,403 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:34:49,539 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:34:50,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:52,004 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:54,005 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:56,006 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:34:58,007 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:00,007 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:02,008 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:04,009 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:04,560 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:35:04,560 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:35:06,010 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:08,011 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:10,011 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:12,012 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:14,013 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:16,014 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:18,015 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:19,545 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:35:19,693 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:35:19,693 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:35:21,017 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:23,017 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:25,018 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:27,019 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:29,020 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:31,020 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:33,021 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:34,829 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:35:34,829 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:35:35,022 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:37,023 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:39,024 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:41,024 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:43,025 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:45,026 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:47,027 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:49,028 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:49,621 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:35:49,961 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:35:49,961 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:35:51,029 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:53,030 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:55,031 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:57,031 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:35:59,032 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:01,033 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:03,034 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:05,035 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:05,092 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:36:05,093 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:36:07,035 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:09,036 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:11,037 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:13,038 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:15,039 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:17,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:19,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:19,692 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:36:20,226 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:36:20,226 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:36:21,041 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:23,042 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:25,043 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:27,044 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:29,045 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:31,045 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:33,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:35,047 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:35,360 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:36:35,361 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:36:37,048 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:39,049 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:41,050 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:43,050 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:45,051 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:47,052 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:49,053 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:49,768 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:36:50,494 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:36:50,495 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:36:51,054 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:53,054 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:55,055 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:57,056 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:36:59,057 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:01,058 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:03,059 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:05,060 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:05,625 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:37:05,625 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:37:07,060 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:09,061 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:11,062 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:13,063 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:15,064 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:17,065 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:19,065 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:19,839 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:37:20,757 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:37:20,758 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:37:21,066 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:23,067 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:25,068 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:27,069 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:29,069 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:31,070 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:33,071 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:35,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:35,898 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:37:35,899 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:37:37,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:39,073 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:41,074 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:43,075 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:45,076 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:47,077 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:49,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:49,915 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:37:51,034 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:37:51,035 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:37:51,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:53,079 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:55,080 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:57,081 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:37:59,082 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:01,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:03,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:05,084 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:06,164 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:38:06,164 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:38:07,085 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:09,086 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:11,087 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:13,088 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:15,089 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:17,089 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:19,090 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:19,992 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:38:21,091 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:21,295 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:38:21,295 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:38:23,092 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:25,092 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:27,093 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:29,094 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:31,095 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:33,096 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:35,096 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:36,425 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:38:36,425 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:38:37,097 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:39,098 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:41,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:43,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:45,100 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:47,101 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:49,102 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:50,066 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:38:51,103 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:51,556 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:38:51,556 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:38:53,103 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:55,104 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:57,105 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:38:59,106 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:01,106 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:03,107 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:05,108 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:06,687 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:39:06,688 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:39:07,109 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:09,109 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:11,110 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:20,139 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:39:21,820 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:39:21,820 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:39:29,346 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:39:29,346 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:39:29,347 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:39:30,117 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:39:31,118 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:32,118 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:33,118 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:35,119 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:36,984 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:39:36,984 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:39:37,121 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:39,122 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:41,122 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:43,123 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:45,124 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:47,125 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:49,125 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:50,212 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:39:51,126 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:52,122 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:39:52,122 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:39:53,127 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:55,128 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:57,129 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:39:59,129 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:01,130 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:03,131 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:05,132 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:07,132 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:07,252 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:40:07,252 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:40:09,133 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:11,134 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:13,135 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:15,135 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:17,136 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:19,137 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:20,287 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:40:21,138 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:22,382 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:40:22,383 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:40:23,139 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:25,139 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:27,140 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:29,141 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:31,142 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:33,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:35,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:37,144 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:37,515 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:40:37,516 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:40:39,145 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:41,146 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:43,146 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:45,147 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:47,148 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:49,149 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:50,361 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:40:51,150 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:52,653 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:40:52,654 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:40:53,151 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:55,151 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:57,152 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:40:59,153 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:01,154 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:03,155 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:05,156 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:07,156 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:07,795 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:41:07,795 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:41:09,157 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:11,158 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:13,159 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:15,159 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:17,160 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:19,161 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:20,434 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:41:21,162 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:22,927 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:41:22,927 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:41:23,163 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:25,163 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:27,164 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:29,165 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:31,166 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:33,167 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:35,167 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:37,168 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:38,063 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:41:38,063 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:41:39,169 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:41,170 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:43,170 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:45,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:47,172 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:49,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:50,511 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:41:51,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:53,174 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:53,197 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:41:53,197 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:41:55,175 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:57,176 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:41:59,177 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:01,177 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:03,178 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:05,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:07,180 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:08,328 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:42:08,328 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:42:09,181 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:11,181 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:13,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:15,183 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:17,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:19,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:20,585 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:42:21,185 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:23,186 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:23,473 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:42:23,473 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:42:25,187 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:27,187 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:29,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:31,189 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:33,190 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:35,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:37,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:38,604 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:42:38,605 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:42:40,192 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:42,193 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:44,194 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:46,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:48,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:50,196 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:50,659 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:42:52,197 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:53,735 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:42:53,735 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:42:54,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:56,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:42:58,199 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:00,200 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:02,201 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:04,202 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:06,203 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:08,203 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:08,869 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:43:08,869 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:43:10,204 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:12,205 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:14,206 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:16,206 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:18,207 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:20,208 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:20,735 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:43:22,208 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:24,010 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:43:24,011 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:43:24,209 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:26,210 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:28,210 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:30,211 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:32,212 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:34,213 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:36,214 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:38,214 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:39,148 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:43:39,148 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:43:40,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:42,216 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:44,217 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:46,218 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:48,218 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:50,219 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:50,812 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:43:52,220 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:54,221 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:54,281 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:43:54,281 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:43:56,221 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:43:58,222 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:09,411 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:44:09,412 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:44:15,476 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:44:15,477 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:44:15,477 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:44:16,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:16,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:44:18,229 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:20,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:20,884 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:44:22,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:24,231 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:24,544 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:44:24,544 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:44:26,232 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:28,233 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:30,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:32,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:34,235 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:36,236 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:38,237 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:39,681 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:44:39,681 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:44:40,238 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:42,239 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:44,239 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:46,240 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:48,241 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:50,242 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:50,958 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:44:52,243 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:54,243 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:54,818 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:44:54,818 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:44:56,244 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:44:58,245 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:00,246 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:02,246 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:04,247 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:06,248 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:08,249 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:09,959 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:45:09,960 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:45:10,250 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:12,251 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:14,252 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:16,253 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:18,254 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:20,255 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:21,033 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:45:22,256 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:24,256 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:25,093 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:45:25,094 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:45:26,257 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:28,258 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:30,259 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:32,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:34,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:36,261 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:38,262 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:40,235 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:45:40,235 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:45:40,263 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:42,264 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:44,264 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:46,265 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:48,266 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:50,266 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:51,107 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:45:52,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:54,268 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:55,366 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:45:55,367 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:45:56,269 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:45:58,269 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:00,270 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:02,271 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:04,272 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:06,272 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:08,273 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:10,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:10,495 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:46:10,496 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:46:12,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:14,275 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:16,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:18,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:20,277 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:21,180 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:46:22,278 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:24,279 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:25,630 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:46:25,631 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:46:26,280 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:28,281 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:30,282 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:32,283 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:34,284 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:36,284 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:38,285 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:40,286 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:40,762 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:46:40,762 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:46:42,287 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:44,288 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:46,289 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:48,290 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:50,290 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:51,253 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:46:52,291 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:54,292 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:55,896 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:46:55,897 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:46:56,293 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:46:58,293 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:00,294 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:02,295 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:04,296 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:06,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:08,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:10,298 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:11,030 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:47:11,030 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:47:12,299 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:14,300 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:16,300 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:18,301 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:20,302 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:21,326 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:47:22,303 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:24,303 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:26,159 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:47:26,159 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:47:26,304 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:28,305 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:30,306 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:32,307 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:34,307 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:36,308 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:38,309 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:40,310 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:41,289 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:47:41,290 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:47:42,311 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:44,312 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:46,313 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:48,314 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:50,314 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:51,400 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:47:52,315 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:54,316 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:56,317 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:47:56,418 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:47:56,419 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:47:58,318 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:00,318 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:02,319 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:04,320 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:06,321 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:08,322 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:10,323 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:11,555 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:48:11,555 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:48:12,323 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:14,324 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:16,325 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:18,325 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:20,326 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:21,473 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:48:22,327 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:24,328 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:26,328 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:26,686 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:48:26,687 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:48:28,329 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:30,330 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:32,331 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:34,332 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:36,332 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:38,333 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:40,334 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:41,816 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:48:41,816 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:48:42,335 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:44,335 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:48:51,546 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:48:56,947 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:48:56,947 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:49:01,614 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:49:01,614 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:49:01,615 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:49:02,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:02,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:49:03,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:04,343 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:06,343 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:08,344 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:10,345 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:12,105 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:49:12,105 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:49:12,346 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:14,347 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:16,347 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:18,348 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:20,349 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:21,620 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:49:22,350 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:24,351 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:26,352 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:27,237 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:49:27,237 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:49:28,353 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:30,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:32,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:34,355 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:36,356 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:38,357 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:40,358 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:42,358 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:42,372 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:49:42,372 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:49:44,359 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:46,360 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:49,362 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:51,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:51,693 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:49:53,363 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:55,364 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:57,365 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:49:57,500 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:49:57,500 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:49:59,366 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:01,366 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:03,367 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:05,368 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:07,369 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:09,370 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:11,370 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:12,658 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:50:12,659 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:50:13,371 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:15,372 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:17,373 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:19,374 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:21,375 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:21,763 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:50:23,376 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:25,377 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:27,377 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:27,787 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:50:27,788 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:50:29,378 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:31,379 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:33,380 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:35,380 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:37,381 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:39,382 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:41,383 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:42,919 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:50:42,920 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:50:43,384 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:45,385 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:47,386 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:49,386 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:51,387 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:51,835 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:50:53,388 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:55,389 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:57,390 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:50:58,050 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:50:58,050 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:50:59,390 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:01,391 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:03,392 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:05,393 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:07,393 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:09,394 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:11,395 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:13,183 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:51:13,183 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:51:13,396 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:15,397 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:17,398 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:19,398 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:21,399 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:21,913 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:51:23,400 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:25,401 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:27,402 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:28,316 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:51:28,317 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:51:29,402 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:31,403 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:33,404 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:35,404 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:37,405 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:39,406 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:41,407 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:43,408 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:43,447 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:51:43,448 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:51:45,408 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:47,409 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:49,410 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:51,411 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:51,986 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:51:53,411 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:55,412 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:57,413 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:51:58,577 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:51:58,577 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:51:59,414 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:01,415 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:03,415 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:05,416 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:07,417 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:09,418 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:11,419 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:13,420 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:13,710 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:52:13,711 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:52:15,421 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:17,422 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:19,422 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:21,423 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:22,063 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:52:23,424 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:25,425 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:27,426 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:28,852 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:52:28,852 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:52:29,427 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:31,428 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:33,429 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:35,430 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:37,431 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:39,432 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:41,433 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:43,434 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:43,981 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:52:43,981 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:52:45,435 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:47,436 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:49,437 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:51,438 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:52,139 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:52:53,439 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:55,440 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:57,440 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:52:59,113 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:52:59,114 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:52:59,441 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:01,442 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:03,443 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:05,443 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:07,444 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:09,445 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:11,446 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:13,447 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:14,244 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:53:14,245 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:53:15,447 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:17,448 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:19,449 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:21,450 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:22,214 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:53:23,451 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:25,451 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:27,452 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:29,376 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:53:29,377 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:53:29,453 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:44,508 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:53:44,508 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:53:47,753 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:53:47,754 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:53:47,754 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:53:48,459 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:53:49,460 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:51,460 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:52,288 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:53:53,461 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:55,462 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:57,463 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:59,464 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:53:59,638 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:53:59,638 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:54:01,465 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:03,466 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:05,466 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:07,467 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:09,468 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:11,469 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:13,470 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:14,810 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:54:14,811 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:54:15,471 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:17,472 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:19,472 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:21,473 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:22,365 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:54:23,474 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:25,475 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:27,476 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:29,477 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:29,942 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:54:29,943 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:54:31,478 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:33,478 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:35,479 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:37,480 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:39,481 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:41,482 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:43,482 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:45,075 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:54:45,076 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:54:45,483 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:47,484 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:49,485 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:51,486 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:52,439 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:54:53,486 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:55,487 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:57,488 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:54:59,489 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:00,209 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:55:00,209 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:55:01,490 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:03,491 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:05,491 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:07,492 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:09,493 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:11,493 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:13,494 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:15,342 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:55:15,343 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:55:15,495 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:17,496 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:19,497 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:21,497 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:22,515 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:55:23,498 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:25,499 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:27,500 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:29,501 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:30,476 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:55:30,476 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:55:31,502 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:33,503 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:35,504 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:37,505 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:39,505 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:41,506 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:43,507 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:45,508 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:45,619 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:55:45,619 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:55:47,509 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:49,510 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:51,511 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:52,593 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:55:53,512 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:55,513 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:57,514 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:55:59,515 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:00,753 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:56:00,753 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:56:01,516 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:03,517 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:05,517 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:07,518 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:09,519 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:11,520 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:13,521 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:15,522 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:15,899 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:56:15,899 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:56:17,523 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:19,524 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:21,525 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:22,670 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:56:23,526 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:25,527 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:27,528 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:29,529 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:31,030 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:56:31,031 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:56:31,530 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:33,531 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:35,532 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:38,533 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:40,534 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:42,535 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:44,536 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:46,163 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:56:46,163 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:56:46,537 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:48,538 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:50,538 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:52,539 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:52,745 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:56:54,540 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:56,541 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:56:58,542 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:00,543 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:01,295 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:57:01,296 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:57:02,544 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:04,544 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:06,545 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:08,546 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:10,547 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:12,548 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:14,548 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:16,427 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:57:16,427 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:57:16,549 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:18,550 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:20,551 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:22,552 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:22,821 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:57:24,553 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:26,554 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:28,555 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:30,556 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:31,556 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:57:31,557 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:57:32,557 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:34,558 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:36,559 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:38,559 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:40,560 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:42,561 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:44,562 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:46,563 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:46,709 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:57:46,709 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:57:48,564 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:50,565 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:52,566 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:52,896 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:57:54,567 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:56,568 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:57:58,568 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:00,569 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:01,844 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:58:01,844 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:58:02,570 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:04,570 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:06,571 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:08,572 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:10,572 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:12,573 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:14,574 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:16,575 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:16,973 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:58:16,974 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:58:22,967 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:58:32,104 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:58:32,104 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:58:33,894 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 06:58:33,894 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 06:58:33,895 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 06:58:34,581 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 06:58:36,582 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:38,583 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:40,584 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:42,585 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:44,586 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:46,587 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:47,245 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:58:47,245 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:58:48,588 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:50,589 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:52,590 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:53,039 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:58:54,591 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:56,591 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:58:58,592 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:00,593 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:02,377 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:59:02,377 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:59:02,594 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:04,595 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:06,596 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:08,596 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:10,597 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:12,598 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:14,599 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:16,600 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:17,512 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:59:17,513 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:59:18,601 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:20,602 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:22,602 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:23,114 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:59:24,603 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:26,604 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:28,605 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:30,605 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:32,606 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:32,642 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:59:32,643 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:59:34,607 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:36,608 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:38,609 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:40,610 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:42,610 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:44,611 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:46,612 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:47,773 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 06:59:47,774 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 06:59:48,613 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:50,614 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:52,615 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:53,189 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 06:59:54,616 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:56,617 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 06:59:58,618 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:00,618 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:02,619 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:02,903 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:00:02,903 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:00:04,620 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:06,620 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:08,621 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:10,622 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:12,623 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:14,624 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:16,625 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:18,035 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:00:18,035 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:00:18,626 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:20,626 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:22,627 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:23,263 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:00:24,628 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:26,629 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:28,630 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:30,631 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:32,631 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:33,168 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:00:33,169 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:00:34,632 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:36,633 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:38,633 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:40,634 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:42,635 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:44,636 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:46,637 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:48,303 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:00:48,303 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:00:48,637 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:50,638 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:52,639 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:53,340 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:00:54,640 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:56,640 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:00:58,641 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:00,642 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:02,643 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:03,433 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:01:03,434 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:01:04,644 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:06,645 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:08,645 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:10,646 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:12,647 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:14,648 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:16,649 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:18,564 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:01:18,564 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:01:18,650 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:20,651 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:22,652 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:23,416 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:01:24,652 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:26,653 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:28,654 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:30,655 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:32,656 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:33,695 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:01:33,696 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:01:34,657 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:36,657 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:38,658 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:40,659 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:42,660 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:44,661 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:46,662 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:48,663 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:48,828 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:01:48,829 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:01:50,664 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:52,664 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:53,490 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:01:54,665 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:56,666 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:01:58,667 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:00,668 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:02,669 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:03,958 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:02:03,958 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:02:04,670 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:06,670 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:08,671 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:10,672 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:12,673 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:14,673 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:16,674 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:18,675 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:19,090 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:02:19,091 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:02:20,676 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:22,677 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:23,564 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:02:24,677 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:26,678 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:28,679 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:30,680 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:32,680 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:34,221 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:02:34,221 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:02:34,681 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:36,682 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:38,683 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:40,684 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:42,684 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:44,685 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:46,686 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:48,687 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:49,356 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:02:49,356 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:02:50,688 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:52,688 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:53,640 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:02:54,689 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:56,690 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:02:58,691 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:00,692 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:02,693 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:04,489 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:03:04,489 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:03:19,621 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:03:19,621 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:03:20,043 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 07:03:20,043 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 07:03:20,043 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 07:03:20,700 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:20,701 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 07:03:22,701 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:23,711 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:03:24,702 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:26,703 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:28,704 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:30,705 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:32,706 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:34,707 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:34,751 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:03:34,751 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:03:36,708 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:38,709 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:40,710 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:42,711 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:44,712 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:46,713 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:48,714 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:49,885 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:03:49,885 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:03:50,715 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:52,715 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:53,786 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:03:54,717 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:56,718 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:03:58,719 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:00,720 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:02,721 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:04,721 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:05,017 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:04:05,017 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:04:06,722 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:08,723 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:10,724 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:12,725 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:14,726 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:16,727 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:18,728 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:20,150 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:04:20,151 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:04:20,729 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:22,730 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:23,861 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:04:24,731 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:26,732 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:28,733 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:31,734 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:33,735 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:35,429 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:04:35,429 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:04:35,736 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:37,737 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:39,738 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:41,739 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:43,740 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:45,741 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:47,742 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:49,743 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:50,566 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:04:50,567 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:04:51,743 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:53,744 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:53,936 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:04:55,745 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:57,746 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:04:59,747 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:01,747 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:03,748 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:05,698 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:05:05,698 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:05:05,749 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:07,750 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:09,751 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:11,752 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:13,752 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:15,753 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:17,754 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:19,755 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:20,900 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:05:20,900 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:05:21,756 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:23,756 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:24,011 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:05:25,757 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:27,758 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:29,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:31,759 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:33,760 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:35,761 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:36,032 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:05:36,032 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:05:37,762 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:39,763 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:41,763 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:43,764 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:45,765 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:47,766 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:49,766 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:51,162 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:05:51,163 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:05:51,767 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:53,768 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:54,087 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:05:55,769 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:57,769 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:05:59,770 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:01,771 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:03,772 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:05,772 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:06,294 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:06:06,295 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:06:07,773 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:09,774 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:11,775 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:13,775 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:15,776 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:17,777 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:19,778 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:21,428 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:06:21,429 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:06:21,778 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:23,779 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:24,161 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:06:25,780 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:27,781 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:29,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:31,782 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:33,783 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:35,784 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:36,560 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:06:36,560 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:06:37,785 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:39,785 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:41,786 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:43,787 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:45,788 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:47,788 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:49,789 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:51,757 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:06:51,757 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:06:51,790 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:53,791 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:54,235 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:06:55,792 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:57,793 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:06:59,794 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:01,795 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:03,795 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:05,796 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:06,952 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:07:06,952 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:07:07,797 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:09,798 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:11,798 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:13,799 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:15,800 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:17,801 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:19,802 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:21,802 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:22,082 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:07:22,082 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:07:23,803 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:24,310 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:07:25,804 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:27,805 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:29,806 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:31,807 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:33,807 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:35,808 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:37,217 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:07:37,217 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:07:37,809 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:39,810 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:41,811 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:43,811 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:45,812 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:47,813 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:49,814 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:07:52,348 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:07:52,348 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:07:54,386 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:08:06,208 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 07:08:06,209 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 07:08:06,209 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 07:08:06,819 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 07:08:07,537 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:08:07,537 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:08:07,820 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:09,821 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:11,821 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:13,822 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:15,823 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:17,823 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:19,824 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:21,825 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:22,670 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:08:22,671 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:08:23,825 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:24,461 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:08:25,826 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:27,827 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:29,827 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:31,828 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:33,829 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:35,830 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:37,830 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:37,834 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:08:37,835 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:08:39,831 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:41,832 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:43,833 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:45,833 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:47,834 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:49,835 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:51,836 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:52,965 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:08:52,966 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:08:53,836 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:54,537 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:08:55,837 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:57,837 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:08:59,838 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:01,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:03,839 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:05,840 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:07,841 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:08,100 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:09:08,100 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:09:09,841 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:11,842 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:13,843 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:15,843 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:17,844 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:19,845 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:21,845 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:23,232 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:09:23,232 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:09:23,846 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:24,613 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:09:25,847 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:27,847 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:29,848 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:31,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:33,849 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:35,850 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:37,851 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:38,367 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:09:38,367 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:09:39,851 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:41,852 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:43,853 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:45,853 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:47,854 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:49,855 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:51,856 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:53,498 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:09:53,498 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:09:53,857 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:54,690 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:09:55,858 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:57,858 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:09:59,859 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:01,860 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:03,860 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:05,861 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:07,861 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:08,627 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:10:08,628 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:10:09,862 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:11,863 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:13,864 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:15,864 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:17,865 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:19,865 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:21,866 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:23,758 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:10:23,758 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:10:23,867 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:24,764 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:10:25,868 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:27,868 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:29,869 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:31,870 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:33,870 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:35,871 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:37,871 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:38,893 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:10:38,893 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:10:39,872 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:41,873 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:43,873 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:45,874 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:47,875 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:49,875 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:51,876 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:53,877 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:54,029 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:10:54,029 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:10:54,840 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:10:55,877 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:57,878 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:10:59,879 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:01,879 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:03,880 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:05,881 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:07,881 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:09,159 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:11:09,159 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:11:09,882 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:11,882 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:13,883 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:15,884 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:17,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:19,885 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:21,886 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:23,887 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:24,292 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:11:24,292 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:11:24,916 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:11:25,887 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:27,888 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:29,889 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:31,889 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:33,890 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:35,891 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:37,891 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:39,434 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:11:39,435 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:11:39,892 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:41,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:43,893 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:45,894 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:47,895 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:49,896 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:51,897 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:53,898 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:54,567 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:11:54,568 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:11:54,992 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:11:55,898 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:57,899 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:11:59,900 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:01,900 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:03,901 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:05,902 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:07,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:09,702 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:12:09,703 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:12:09,903 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:11,904 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:13,905 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:15,905 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:17,906 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:19,907 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:21,907 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:23,908 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:24,873 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:12:24,873 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:12:25,067 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:12:25,909 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:27,909 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:29,910 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:31,911 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:33,911 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:40,006 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:12:40,007 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:12:52,350 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 07:12:52,351 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 07:12:52,351 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 07:12:52,918 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 07:12:53,918 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:55,140 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:12:55,204 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:12:55,204 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:12:55,919 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:12:57,920 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:00,921 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:02,921 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:04,922 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:06,923 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:08,924 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:10,335 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:13:10,336 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:13:10,925 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:12,926 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:14,927 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:16,927 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:18,928 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:20,929 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:22,930 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:24,931 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:25,208 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:13:25,465 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:13:25,465 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:13:26,931 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:28,932 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:30,933 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:32,934 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:34,934 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:36,935 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:38,936 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:40,595 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:13:40,596 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:13:40,937 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:42,937 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:44,938 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:46,939 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:48,940 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:50,941 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:52,941 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:54,942 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:55,279 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:13:55,728 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:13:55,728 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:13:56,943 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:13:58,944 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:00,945 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:02,945 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:04,946 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:06,947 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:08,947 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:10,855 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:14:10,855 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:14:10,948 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:12,949 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:14,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:16,950 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:18,951 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:20,952 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:22,953 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:24,954 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:25,348 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:14:25,984 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:14:25,984 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:14:26,954 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:28,955 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:30,956 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:32,957 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:34,958 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:36,958 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:38,959 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:40,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:41,115 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:14:41,115 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:14:42,961 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:44,961 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:46,962 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:48,963 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:50,964 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:52,965 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:54,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:55,423 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:14:56,245 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:14:56,246 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:14:56,966 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:14:58,967 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:00,968 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:02,969 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:04,970 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:06,970 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:08,971 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:10,972 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:11,375 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:15:11,376 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:15:12,972 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:14,973 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:16,974 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:18,975 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:20,976 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:22,976 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:24,977 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:25,497 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:15:26,504 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:15:26,504 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:15:26,978 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:28,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:30,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:32,980 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:34,981 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:36,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:38,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:40,983 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:41,635 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:15:41,635 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:15:42,984 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:44,985 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:46,985 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:48,986 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:50,987 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:52,987 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:54,988 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:55,567 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:15:56,765 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:15:56,765 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:15:56,989 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:15:58,990 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:00,990 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:02,991 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:04,992 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:06,992 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:08,993 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:10,994 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:11,894 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:16:11,894 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:16:12,995 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:14,996 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:16,997 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:18,997 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:20,998 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:22,999 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:25,000 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:25,640 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:16:27,000 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:27,025 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:16:27,025 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:16:29,001 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:31,002 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:33,003 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:35,004 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:37,004 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:39,005 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:41,006 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:42,156 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:16:42,157 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:16:43,007 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:45,008 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:47,008 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:49,009 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:51,010 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:53,010 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:55,011 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:55,711 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:16:57,012 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:16:57,287 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:16:57,287 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:16:59,012 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:01,013 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:03,014 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:05,015 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:07,016 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:09,016 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:11,017 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:12,418 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:17:12,418 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:17:13,018 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:15,019 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:17,019 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:19,020 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:21,021 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:25,776 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:17:27,548 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:17:27,549 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:17:38,514 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 07:17:38,514 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 07:17:38,515 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 07:17:39,027 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 07:17:41,027 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:42,740 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:17:42,740 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:17:43,028 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:45,029 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:47,030 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:49,031 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:51,032 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:53,033 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:55,034 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:55,840 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:17:57,034 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:17:57,871 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:17:57,872 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:17:59,035 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:01,036 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:03,037 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:05,037 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:07,038 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:09,039 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:11,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:13,002 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:18:13,003 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:18:13,040 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:15,041 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:17,042 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:19,043 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:21,044 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:23,045 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:25,045 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:25,908 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:18:27,046 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:28,134 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:18:28,135 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:18:29,047 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:31,048 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:33,049 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:35,049 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:37,050 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:39,051 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:41,051 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:43,052 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:43,264 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:18:43,264 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:18:45,053 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:47,054 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:49,054 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:51,055 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:53,056 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:55,057 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:55,980 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:18:57,057 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:18:58,394 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:18:58,395 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:18:59,058 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:01,059 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:03,060 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:05,060 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:07,061 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:09,062 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:11,062 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:13,063 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:13,536 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:19:13,537 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:19:15,064 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:17,064 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:19,065 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:21,066 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:23,067 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:25,068 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:26,054 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:19:27,068 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:28,668 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:19:28,669 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:19:29,069 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:31,070 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:33,070 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:35,071 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:37,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:39,072 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:41,073 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:43,074 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:43,800 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:19:43,801 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:19:45,075 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:47,075 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:49,076 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:51,077 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:53,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:55,078 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:56,129 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:19:58,080 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:19:58,934 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:19:58,934 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:20:00,080 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:02,081 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:04,082 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:06,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:08,083 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:10,084 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:12,085 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:14,070 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:20:14,071 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:20:14,086 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:16,087 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:18,088 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:20,089 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:22,090 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:24,091 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:26,091 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:26,204 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:20:28,092 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:29,205 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:20:29,205 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:20:30,093 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:32,094 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:34,095 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:36,095 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:38,096 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:40,097 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:42,098 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:44,099 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:44,336 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:20:44,337 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:20:46,100 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:48,100 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:50,101 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:52,102 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:54,103 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:56,104 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:56,276 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:20:58,104 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:20:59,465 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:20:59,465 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:21:00,105 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:02,106 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:04,107 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:06,108 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:08,109 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:10,110 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:12,111 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:14,112 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:14,596 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:21:14,597 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:21:16,112 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:18,113 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:20,114 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:22,115 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:24,116 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:26,117 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:26,347 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:21:28,118 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:29,730 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:21:29,730 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:21:30,119 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:32,119 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:34,120 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:36,121 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:38,122 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:40,123 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:42,123 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:44,124 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:44,858 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:21:44,859 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:21:46,125 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:48,126 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:50,127 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:52,128 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:54,129 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:56,129 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:56,418 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:21:58,130 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:21:59,988 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:21:59,989 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:22:00,131 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:02,132 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:04,133 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:06,133 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:08,134 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:15,119 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:22:15,120 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:22:24,645 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 07:22:24,645 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 07:22:24,646 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 07:22:25,141 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 07:22:26,141 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:26,491 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:22:27,142 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:28,142 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:30,143 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:30,254 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:22:30,255 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:22:32,144 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:34,145 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:36,146 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:38,147 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:40,148 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:42,149 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:44,150 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:45,418 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:22:45,419 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:22:46,150 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:48,151 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:50,152 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:52,153 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:54,154 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:56,154 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:22:56,566 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:22:58,155 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:00,156 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:00,549 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:23:00,550 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:23:02,157 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:04,158 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:06,158 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:08,159 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:10,160 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:12,161 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:14,162 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:15,683 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:23:15,683 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:23:16,162 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:18,163 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:20,164 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:22,165 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:24,166 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:26,166 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:26,638 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:23:28,167 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:30,168 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:30,814 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:23:30,814 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:23:32,169 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:34,170 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:36,171 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:38,172 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:40,172 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:42,173 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:44,174 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:45,946 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:23:45,947 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:23:46,175 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:48,176 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:50,177 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:52,178 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:54,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:56,179 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:23:56,712 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:23:58,180 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:00,181 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:01,079 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:24:01,079 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:24:02,182 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:04,183 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:06,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:08,184 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:10,185 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:12,186 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:14,187 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:16,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:16,216 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:24:16,216 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:24:18,188 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:20,189 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:22,190 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:24,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:26,191 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:26,786 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:24:28,192 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:30,193 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:31,347 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:24:31,347 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:24:32,194 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:34,194 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:36,195 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:38,196 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:40,197 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:42,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:44,198 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:46,199 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:46,482 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:24:46,483 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:24:48,200 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:50,201 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:52,201 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:54,202 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:56,203 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:24:56,859 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:24:58,204 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:00,204 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:01,615 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:25:01,615 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:25:02,205 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:04,206 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:06,207 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:08,207 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:10,208 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:12,209 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:14,210 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:16,211 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:16,744 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:25:16,745 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:25:18,212 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:20,213 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:22,214 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:24,215 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:26,216 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:26,933 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:25:28,216 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:30,217 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:31,875 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:25:31,875 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:25:32,218 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:34,219 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:36,220 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:38,220 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:40,221 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:42,222 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:44,223 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:46,223 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:47,017 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:25:47,017 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:25:48,224 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:50,225 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:52,225 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:54,226 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:56,227 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:25:57,008 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:25:58,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:00,228 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:02,149 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:26:02,149 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:26:02,229 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:04,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:06,230 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:08,231 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:10,232 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:12,233 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:14,233 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:16,234 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:17,281 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:26:17,281 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:26:18,235 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:20,236 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:22,236 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:24,237 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:26,238 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:27,084 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:26:28,239 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:30,240 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:32,241 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:32,413 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:26:32,414 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:26:34,241 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:36,242 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:38,243 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:40,244 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:42,244 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:44,245 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:46,246 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:47,545 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:26:47,545 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:26:48,246 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:50,247 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:52,248 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:26:57,162 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:27:02,684 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:27:02,684 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:27:10,846 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 07:27:10,846 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 07:27:10,847 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 07:27:11,254 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 07:27:13,255 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:15,255 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:17,256 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:17,816 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:27:17,816 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:27:19,257 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:21,258 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:23,258 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:25,259 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:27,233 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:27:27,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:29,260 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:31,261 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:32,948 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:27:32,948 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:27:33,262 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:35,263 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:37,264 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:39,265 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:41,266 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:43,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:45,267 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:47,268 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:48,078 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:27:48,078 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:27:49,269 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:51,270 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:53,270 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:55,271 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:57,272 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:27:57,307 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:27:59,273 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:01,273 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:03,209 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:28:03,209 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:28:03,274 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:05,275 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:07,276 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:09,277 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:11,278 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:13,278 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:15,279 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:17,280 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:18,340 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:28:18,341 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:28:19,281 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:21,282 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:23,283 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:25,283 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:27,284 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:27,380 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:28:29,285 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:31,286 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:33,286 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:33,477 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:28:33,478 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:28:35,287 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:37,288 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:39,289 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:41,290 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:43,291 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:45,291 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:47,292 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:48,606 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:28:48,607 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:28:49,293 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:51,294 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:53,294 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:55,295 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:57,296 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:28:57,454 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:28:59,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:01,297 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:03,298 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:03,737 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:29:03,737 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:29:05,299 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:07,300 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:09,301 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:11,301 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:13,302 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:15,303 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:17,304 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:18,869 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:29:18,869 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:29:19,305 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:21,306 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:23,306 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:25,307 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:27,308 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:27,527 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:29:29,309 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:31,309 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:33,310 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:34,001 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:29:34,001 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:29:35,311 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:37,312 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:39,313 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:41,313 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:43,314 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:45,315 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:47,316 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:49,135 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:29:49,135 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:29:49,316 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:51,317 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:53,318 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:55,318 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:57,319 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:29:57,601 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:29:59,320 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:01,320 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:03,321 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:04,263 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:30:04,264 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:30:05,321 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:07,322 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:09,323 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:11,323 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:13,324 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:15,325 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:17,325 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:19,326 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:19,396 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:30:19,396 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:30:21,327 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:23,327 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:25,328 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:27,329 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:27,673 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:30:29,329 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:31,330 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:33,331 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:34,526 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:30:34,526 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:30:35,331 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:37,332 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:39,333 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:41,333 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:43,334 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:45,335 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:47,335 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:49,336 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:49,654 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:30:49,655 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:30:51,337 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:53,338 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:55,339 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:57,339 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:30:57,745 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:30:59,340 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:01,341 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:03,342 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:04,787 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:31:04,787 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:31:05,343 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:07,343 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:09,344 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:11,345 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:13,345 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:15,346 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:17,346 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:19,347 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:19,919 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:31:19,919 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:31:21,348 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:23,348 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:25,349 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:27,350 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:27,816 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:31:29,350 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:31,351 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:33,352 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:35,051 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:31:35,052 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:31:35,352 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:37,353 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:39,354 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:50,184 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 07:31:50,184 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 07:31:56,969 DEBUG SenderThread:332390 [sender.py:send():179] send: history +2021-07-13 07:31:56,969 DEBUG SenderThread:332390 [sender.py:send():179] send: summary +2021-07-13 07:31:56,969 INFO SenderThread:332390 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 07:31:57,361 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json +2021-07-13 07:31:57,889 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 07:31:58,361 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 07:31:59,362 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log