diff --git "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" --- "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" +++ "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" @@ -12406,3 +12406,627 @@ 2022-02-02 03:42:11,886 DEBUG SenderThread:38198 [sender.py:send():234] send: stats 2022-02-02 03:42:15,947 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status 2022-02-02 03:42:15,947 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:42:24,740 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:26,741 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:28,743 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:31,299 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:42:31,299 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:42:32,745 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:34,747 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:36,748 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:38,749 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:40,751 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:42,134 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:42:42,752 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:44,753 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:46,456 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:42:46,456 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:42:46,754 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:48,756 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:50,757 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:52,759 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:54,760 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:56,761 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:42:58,763 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:00,764 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:01,746 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:43:01,746 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:43:02,765 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:04,767 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:06,768 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:08,770 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:10,772 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:12,402 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:43:12,773 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:14,775 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:16,777 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:17,081 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:43:17,082 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:43:18,780 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:20,782 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:22,783 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:24,786 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:26,787 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:28,789 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:30,791 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:32,399 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:43:32,399 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:43:32,793 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:34,794 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:36,796 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:39,799 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:41,800 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:42,647 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:43:43,803 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:45,805 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:47,677 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:43:47,678 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:43:47,807 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:49,809 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:51,810 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:53,811 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:55,813 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:57,815 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:43:59,817 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:01,819 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:02,935 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:44:02,936 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:44:03,821 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:05,823 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:07,824 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:09,826 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:11,827 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:12,908 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:44:13,829 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:15,831 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:17,832 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:18,114 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:44:18,115 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:44:19,833 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:21,834 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:23,836 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:24,590 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:44:24,648 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:44:24,677 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:44:24,837 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:44:25,837 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:27,839 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:29,840 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:31,842 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:33,378 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:44:33,379 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:44:33,843 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:35,844 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:37,845 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:39,847 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:41,848 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:43,221 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:44:43,849 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:45,851 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:47,852 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:48,633 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:44:48,634 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:44:49,853 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:52,855 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:54,857 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:56,860 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:44:58,861 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:00,863 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:02,864 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:03,884 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:45:03,885 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:45:04,866 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:06,867 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:08,868 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:10,869 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:12,870 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:13,519 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:45:14,872 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:16,873 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:18,874 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:19,134 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:45:19,135 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:45:20,876 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:22,877 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:24,879 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:26,880 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:28,881 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:30,883 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:32,884 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:34,412 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:45:34,413 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:45:36,886 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:38,887 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:40,889 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:42,890 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:43,789 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:45:44,891 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:46,892 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:48,894 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:49,713 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:45:49,713 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:45:50,895 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:52,897 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:54,898 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:56,899 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:45:58,900 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:00,902 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:02,902 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:04,904 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:04,954 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:46:04,955 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:46:06,905 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:08,907 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:10,908 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:12,909 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:14,090 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:46:14,909 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:16,911 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:18,912 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:20,108 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:46:20,108 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:46:20,913 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:22,914 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:24,267 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:46:24,358 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:46:24,403 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:46:24,915 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:24,916 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:46:25,916 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:26,917 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:28,918 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:30,920 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:33,922 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:35,366 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:46:35,367 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:46:37,924 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:39,925 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:41,926 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:43,927 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:44,404 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:46:45,929 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:47,930 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:49,931 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:50,539 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:46:50,539 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:46:51,933 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:53,934 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:55,936 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:57,937 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:46:59,938 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:01,939 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:03,941 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:05,691 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:47:05,692 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:47:05,942 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:07,944 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:09,945 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:11,946 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:13,947 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:14,664 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:47:15,949 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:17,951 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:19,952 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:20,856 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:47:20,857 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:47:21,953 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:23,955 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:25,956 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:27,957 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:29,958 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:33,961 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:35,962 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:36,062 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:47:36,063 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:47:37,963 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:40,965 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:42,967 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:44,939 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:47:44,969 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:46,971 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:48,972 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:50,974 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:51,277 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:47:51,277 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:47:52,975 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:54,976 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:56,978 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:47:58,979 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:00,980 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:02,982 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:04,983 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:06,529 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:48:06,530 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:48:06,984 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:08,985 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:10,987 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:12,988 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:14,989 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:15,167 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:48:16,990 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:18,991 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:20,993 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:21,702 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:48:21,703 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:48:22,994 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:24,914 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:48:24,975 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:48:25,006 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:25,006 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:48:25,007 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:48:27,007 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:29,009 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:31,010 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:35,013 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:37,003 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:48:37,005 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:48:37,015 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:39,017 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:41,018 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:43,019 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:45,021 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:45,428 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:48:47,022 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:49,024 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:51,025 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:52,241 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:48:52,241 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:48:53,026 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:55,027 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:57,029 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:48:59,030 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:01,034 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:03,035 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:05,037 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:07,558 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:49:07,558 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:49:09,040 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:11,042 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:13,044 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:15,046 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:15,639 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:49:17,048 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:19,049 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:21,051 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:22,804 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:49:22,804 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:49:24,053 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:26,055 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:28,057 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:30,059 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:32,061 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:34,063 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:36,065 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:38,059 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:49:38,059 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:49:38,067 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:40,069 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:42,070 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:44,072 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:45,967 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:49:46,075 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:48,077 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:50,078 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:52,080 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:53,270 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:49:53,270 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:49:54,082 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:56,083 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:49:58,085 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:00,087 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:02,089 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:04,091 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:08,094 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:08,475 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:50:08,476 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:50:10,096 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:12,098 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:14,100 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:16,101 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:16,350 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:50:18,102 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:20,104 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:22,105 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:23,644 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:50:23,644 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:50:24,106 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:26,108 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:28,109 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:30,110 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:30,762 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:50:30,876 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:50:30,909 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:50:31,111 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:50:32,112 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:33,113 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:34,113 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:36,117 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:38,118 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:38,898 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:50:38,900 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:50:40,120 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:42,121 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:44,122 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:46,123 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:46,622 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:50:48,134 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:50,135 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:52,136 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:54,118 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:50:54,119 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:50:54,137 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:57,139 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:50:59,141 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:01,142 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:03,143 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:05,144 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:07,146 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:09,147 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:09,293 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:51:09,293 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:51:13,149 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:15,151 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:16,863 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:51:17,152 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:19,153 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:21,154 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:23,156 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:24,458 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:51:24,459 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:51:25,158 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:27,160 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:29,162 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:31,164 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:33,166 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:35,168 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:37,170 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:39,172 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:39,624 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:51:39,625 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:51:41,174 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:43,176 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:45,177 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:47,125 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:51:47,178 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:49,180 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:51,181 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:53,183 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:54,791 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:51:54,792 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:51:55,183 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:57,185 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:51:59,186 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:01,187 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:03,188 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:05,190 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:09,192 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:10,038 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:52:10,039 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:52:11,193 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:13,194 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:15,196 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:17,197 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:17,372 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:52:19,198 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:21,200 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:23,201 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:25,203 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:25,206 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:52:25,206 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:52:27,205 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:29,207 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:30,663 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:52:30,761 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:52:30,807 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:52:31,208 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:31,208 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:52:33,210 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:34,211 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:36,213 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:38,215 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:40,217 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:40,367 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:52:40,368 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:52:42,219 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:44,221 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:46,223 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:47,667 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:52:48,225 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:50,227 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:52,229 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:54,231 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:55,536 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:52:55,537 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:52:56,232 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:52:58,234 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:00,236 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:02,238 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:04,240 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:06,242 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:08,244 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:10,246 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:10,872 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:53:10,873 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:53:12,248 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:14,250 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:16,252 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:17,916 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:53:18,254 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:20,256 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:22,258 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:25,260 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:26,099 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:53:26,100 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:53:27,261 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:29,263 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:31,264 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:33,265 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:35,266 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:37,268 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:39,269 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:41,271 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:41,273 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:53:41,274 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:53:43,272 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:45,273 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:47,274 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:48,214 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:53:49,276 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:51,277 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:53,278 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:55,279 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:56,439 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:53:56,440 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:53:57,281 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:53:59,282 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:01,283 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:03,285 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:05,286 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:07,287 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:09,289 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:11,290 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:11,596 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:54:11,597 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:54:13,291 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:15,292 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:17,294 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:18,488 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:54:19,295 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:21,296 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:23,298 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:25,299 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:26,921 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:54:26,922 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:54:27,300 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:29,302 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:31,303 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:33,304 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:36,306 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:38,308 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:40,309 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:42,278 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:54:42,278 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:54:42,310 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:44,311 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:46,313 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:48,314 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:48,709 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:54:50,315 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:52,316 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:54,317 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:56,319 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:54:57,441 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:54:57,441 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:54:58,322 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:00,323 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:02,324 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:04,326 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:06,327 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:08,330 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:10,332 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:12,333 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:12,655 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:55:12,656 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:55:14,335 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:16,338 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:18,339 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:18,920 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:55:20,341 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:22,343 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:24,345 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:26,347 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:27,859 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:55:27,859 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:55:28,349 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:30,351 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:32,353 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:34,354 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:36,356 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:38,358 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:40,360 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:42,362 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:43,037 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:55:43,038 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:55:44,364 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:47,366 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:49,222 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:55:49,368 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:51,370 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:53,371 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:55,372 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:57,373 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:55:58,251 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:55:58,251 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:55:59,373 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:01,375 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:03,376 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:05,378 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:07,379 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:09,380 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:11,382 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:13,383 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:13,415 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:56:13,415 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:56:15,384 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:17,386 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:19,387 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:19,482 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:56:21,389 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:23,391 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:25,393 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:27,395 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:28,715 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:56:28,716 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:56:29,397 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:31,399 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:33,401 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:35,403 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:37,405 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:39,407 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:41,409 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:43,411 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:43,889 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:56:43,889 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:56:45,413 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:47,414 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:49,416 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:49,751 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:56:51,419 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:53,421 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:55,423 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:57,425 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:56:59,138 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:56:59,139 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:56:59,427 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:57:01,429 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:57:14,331 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:57:14,332 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:57:19,972 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:57:29,541 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:57:29,541 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:57:44,755 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:57:44,756 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:57:50,185 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:57:59,979 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:57:59,980 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:58:00,494 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:58:00,508 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:58:00,538 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:58:01,464 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:58:02,465 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:58:03,467 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:58:04,468 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:58:12,473 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:58:15,172 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:58:15,173 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:58:20,421 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:58:30,340 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:58:30,341 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:58:45,507 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:58:45,507 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:58:50,660 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:59:00,686 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:59:00,687 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:59:15,872 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:59:15,873 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:59:20,952 DEBUG SenderThread:38198 [sender.py:send():234] send: stats