diff --git "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" --- "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" +++ "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" @@ -11788,3 +11788,621 @@ 2022-02-02 03:25:02,156 DEBUG SenderThread:38198 [sender.py:send():234] send: stats 2022-02-02 03:25:16,307 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status 2022-02-02 03:25:16,308 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:25:31,483 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:25:31,484 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:25:32,381 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:25:33,922 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:35,923 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:37,924 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:39,925 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:41,927 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:43,928 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:45,930 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:46,660 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:25:46,660 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:25:47,931 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:49,932 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:51,934 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:53,935 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:55,936 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:57,938 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:25:59,939 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:01,868 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:26:01,868 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:26:02,603 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:26:03,942 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:05,944 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:07,946 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:09,948 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:11,949 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:13,951 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:15,953 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:17,051 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:26:17,051 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:26:17,955 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:19,958 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:21,960 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:23,962 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:25,963 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:27,965 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:29,967 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:31,968 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:32,410 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:26:32,410 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:26:32,964 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:26:33,969 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:35,971 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:37,972 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:40,974 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:42,975 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:44,977 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:46,978 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:47,733 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:26:47,734 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:26:48,980 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:50,982 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:52,983 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:54,985 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:56,987 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:26:58,989 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:00,990 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:02,971 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:27:02,972 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:27:02,992 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:03,371 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:27:04,994 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:06,996 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:08,998 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:11,000 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:13,002 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:15,004 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:17,006 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:18,369 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:27:18,369 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:27:19,007 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:21,009 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:23,011 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:25,013 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:26,702 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:27:26,781 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:27:26,827 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:27:27,015 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:27,016 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:27:28,017 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:29,018 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:31,020 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:33,021 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:33,551 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:27:33,552 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:27:33,709 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:27:35,022 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:39,025 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:41,026 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:43,027 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:45,029 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:47,030 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:48,711 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:27:48,712 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:27:49,032 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:51,033 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:53,034 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:55,035 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:27:58,037 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:00,039 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:02,040 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:03,889 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:28:03,889 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:28:04,042 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:04,056 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:28:06,043 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:08,043 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:10,045 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:12,046 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:14,048 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:16,049 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:18,051 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:19,095 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:28:19,096 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:28:20,052 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:22,053 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:24,054 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:26,055 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:28,057 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:30,058 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:32,059 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:34,281 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:28:34,293 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:28:34,293 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:28:36,062 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:38,063 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:40,064 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:42,066 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:44,067 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:46,068 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:48,070 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:49,459 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:28:49,460 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:28:50,071 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:52,072 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:54,074 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:56,075 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:28:58,076 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:00,078 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:02,079 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:04,080 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:04,634 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:29:04,634 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:29:04,799 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:29:06,081 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:08,083 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:10,084 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:12,085 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:14,086 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:16,087 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:18,088 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:19,802 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:29:19,803 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:29:20,090 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:22,091 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:24,092 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:25,340 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:29:25,424 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:29:25,452 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:29:26,093 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:26,094 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:29:28,094 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:30,095 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:32,096 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:35,005 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:29:35,006 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:29:35,172 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:29:36,098 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:38,100 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:40,101 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:42,103 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:44,105 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:46,107 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:48,109 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:50,110 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:50,174 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:29:50,175 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:29:52,112 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:54,114 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:56,116 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:29:58,117 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:00,119 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:02,121 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:05,123 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:05,287 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:30:05,389 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:30:05,389 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:30:07,125 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:09,127 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:11,129 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:13,130 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:15,133 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:17,135 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:19,137 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:20,546 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:30:20,547 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:30:21,139 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:23,141 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:25,143 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:27,145 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:29,147 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:31,148 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:33,150 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:35,152 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:35,524 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:30:35,714 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:30:35,715 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:30:39,155 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:41,157 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:43,159 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:45,161 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:47,163 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:49,166 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:50,884 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:30:50,885 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:30:51,169 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:53,170 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:55,172 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:57,173 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:30:59,175 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:01,176 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:03,178 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:05,179 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:05,835 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:31:06,110 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:31:06,110 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:31:07,181 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:09,183 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:11,185 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:13,186 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:15,188 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:17,190 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:19,192 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:21,193 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:21,306 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:31:21,307 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:31:23,195 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:25,196 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:25,354 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:31:25,415 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:31:25,457 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:31:26,197 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:26,197 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:31:27,198 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:29,199 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:31,200 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:35,203 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:36,185 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:31:36,634 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:31:36,635 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:31:37,208 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:39,209 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:41,211 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:43,213 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:45,215 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:47,216 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:49,218 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:51,220 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:51,803 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:31:51,803 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:31:53,222 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:55,225 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:31:58,227 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:00,229 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:02,232 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:04,234 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:06,236 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:06,467 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:32:07,050 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:32:07,051 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:32:08,238 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:10,240 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:12,242 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:14,244 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:16,246 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:18,249 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:20,251 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:22,220 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:32:22,220 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:32:22,253 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:24,254 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:26,255 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:28,257 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:30,258 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:32,260 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:36,263 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:36,771 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:32:37,389 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:32:37,390 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:32:38,264 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:40,266 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:42,267 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:44,269 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:46,270 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:48,274 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:50,275 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:52,277 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:52,561 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:32:52,562 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:32:54,278 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:56,279 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:32:58,281 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:00,282 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:02,283 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:04,285 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:06,287 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:07,019 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:33:07,864 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:33:07,864 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:33:08,289 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:10,291 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:12,292 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:14,294 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:16,296 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:18,298 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:20,300 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:22,302 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:23,096 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:33:23,096 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:33:24,304 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:26,306 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:26,698 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:33:26,802 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:33:26,835 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:33:27,307 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:33:28,308 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:29,309 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:30,310 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:32,312 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:36,315 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:37,277 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:33:38,248 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:33:38,248 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:33:38,316 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:40,318 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:42,319 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:44,321 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:46,323 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:48,325 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:51,328 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:53,330 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:53,481 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:33:53,481 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:33:55,331 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:57,333 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:33:59,335 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:01,337 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:03,339 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:05,341 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:07,343 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:07,546 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:34:08,648 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:34:08,649 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:34:09,345 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:11,347 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:13,349 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:15,350 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:17,352 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:19,353 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:21,354 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:23,355 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:23,872 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:34:23,872 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:34:25,356 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:27,358 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:29,358 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:31,360 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:35,362 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:37,364 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:37,878 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:34:39,070 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:34:39,071 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:34:39,365 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:41,366 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:43,367 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:45,368 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:47,370 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:49,371 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:51,372 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:53,374 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:54,312 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:34:54,313 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:34:55,375 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:57,376 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:34:59,378 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:01,379 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:03,380 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:05,381 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:07,383 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:08,174 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:35:09,384 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:09,560 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:35:09,560 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:35:12,386 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:14,388 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:16,389 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:18,390 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:20,391 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:22,393 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:24,395 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:24,821 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:35:24,821 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:35:25,553 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:35:25,638 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:35:25,674 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:35:26,396 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:26,397 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:35:27,398 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:28,399 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:30,401 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:32,403 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:34,404 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:36,406 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:38,407 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:38,517 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:35:40,089 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:35:40,090 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:35:40,408 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:42,410 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:44,411 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:46,412 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:48,413 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:50,415 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:52,416 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:54,418 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:55,429 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:35:55,431 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:35:56,420 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:35:58,421 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:00,422 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:02,424 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:04,425 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:06,427 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:08,428 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:08,753 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:36:10,430 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:10,583 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:36:10,583 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:36:12,431 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:14,432 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:17,434 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:19,436 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:21,438 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:23,440 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:25,442 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:25,916 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:36:25,916 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:36:27,443 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:29,445 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:31,447 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:33,449 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:35,451 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:37,453 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:39,098 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:36:39,455 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:41,230 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:36:41,230 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:36:41,457 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:43,459 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:45,461 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:47,463 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:49,465 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:51,467 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:53,469 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:55,471 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:56,418 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:36:56,418 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:36:57,473 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:36:59,475 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:01,476 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:03,477 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:05,479 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:07,480 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:09,406 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:37:09,481 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:11,482 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:11,604 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:37:11,604 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:37:13,484 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:15,485 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:17,486 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:19,487 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:22,489 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:24,491 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:26,493 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:26,948 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:37:26,949 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:37:28,495 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:30,497 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:32,499 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:34,501 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:36,503 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:38,504 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:39,699 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:37:40,506 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:42,188 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:37:42,188 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:37:42,508 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:44,510 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:46,511 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:48,513 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:50,515 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:52,517 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:54,519 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:56,521 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:37:57,368 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:37:57,368 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:37:58,522 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:00,524 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:02,526 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:04,528 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:06,530 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:08,532 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:09,921 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:38:10,534 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:12,536 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:12,634 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:38:12,635 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:38:14,538 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:16,539 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:18,541 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:20,543 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:22,545 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:24,547 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:26,549 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:27,867 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:38:27,868 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:38:28,550 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:31,553 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:33,555 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:35,557 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:37,559 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:39,561 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:40,151 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:38:41,563 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:43,034 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:38:43,035 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:38:43,565 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:45,567 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:47,569 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:49,570 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:51,572 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:53,574 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:55,576 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:57,578 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:38:58,274 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:38:58,275 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:38:59,580 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:01,582 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:03,584 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:05,586 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:07,588 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:09,590 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:10,374 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:39:11,592 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:13,537 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:39:13,538 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:39:13,595 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:15,597 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:17,598 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:19,600 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:21,602 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:23,604 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:25,605 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:27,607 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:28,724 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:39:28,725 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:39:29,609 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:31,611 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:33,613 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:35,615 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:37,617 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:39,618 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:40,671 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:39:41,621 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:43,622 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:44,031 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:39:44,032 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:39:45,624 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:47,626 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:49,627 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:51,629 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:53,631 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:56,633 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:39:59,236 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:39:59,236 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:40:10,893 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:40:14,445 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:40:14,446 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:40:29,674 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:40:29,674 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:40:41,145 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:40:44,877 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:40:44,878 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:40:54,302 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 03:40:54,307 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 03:40:54,339 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 03:40:54,668 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:40:54,668 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 03:40:56,670 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:40:58,672 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:41:00,072 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:41:00,073 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:41:06,677 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:41:11,376 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:41:15,240 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:41:15,241 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:41:30,402 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:41:30,402 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:41:41,610 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:41:45,586 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:41:45,587 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:42:00,776 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:42:00,776 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:42:11,886 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:42:15,947 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:42:15,947 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status