diff --git "a/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" "b/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" --- "a/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" +++ "b/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" @@ -13900,3 +13900,796 @@ 2022-01-29 19:48:56,565 DEBUG SenderThread:13445 [sender.py:send():234] send: stats 2022-01-29 19:48:57,429 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status 2022-01-29 19:48:57,430 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:49:06,069 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:08,070 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:10,071 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:12,072 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:12,699 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:49:12,700 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:49:16,074 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:20,077 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:24,079 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:26,081 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:26,818 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:49:27,941 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:49:27,941 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:49:30,083 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:35,087 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:37,088 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:41,090 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:43,163 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:49:43,163 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:49:45,093 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:47,095 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:51,097 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:53,099 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:57,057 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:49:57,102 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:49:58,452 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:49:58,452 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:49:59,103 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:03,106 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:05,107 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:09,110 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:11,111 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:13,649 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:50:13,650 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:50:15,114 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:17,116 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:21,118 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:23,119 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:25,121 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:27,305 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:50:28,868 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:50:28,868 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:50:29,123 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:31,125 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:33,126 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:37,129 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:39,131 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:41,132 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:43,134 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:44,051 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:50:44,052 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:50:47,136 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:49,138 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:51,140 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:54,142 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:56,144 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:50:57,532 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:50:59,224 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:50:59,224 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:51:00,146 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:02,147 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:04,148 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:06,150 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:08,151 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:10,153 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:12,155 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:14,156 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:14,458 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:51:14,458 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:51:16,158 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:18,159 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:20,160 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:22,161 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:24,162 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:26,164 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:27,742 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:51:29,659 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:51:29,660 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:51:30,166 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:32,167 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:36,170 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:40,173 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:44,176 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:44,914 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:51:44,915 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:51:46,178 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:50,181 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:52,183 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:56,186 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:51:57,980 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:51:58,188 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:00,182 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:52:00,183 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:52:02,192 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:06,195 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:08,197 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:13,200 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:15,202 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:15,360 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:52:15,360 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:52:19,204 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:21,206 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:25,208 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:27,209 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:28,214 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:52:30,529 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:52:30,530 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:52:31,216 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:33,217 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:35,219 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:39,221 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:41,223 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:43,224 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:45,782 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:52:45,783 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:52:47,226 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:49,228 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:51,230 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:53,232 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:57,234 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:52:58,425 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:52:59,236 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:01,039 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:53:01,039 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:53:01,238 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:03,239 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:07,241 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:09,242 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:11,244 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:13,246 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:15,248 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:16,305 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:53:16,305 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:53:17,250 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:19,251 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:22,253 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:24,255 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:26,256 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:28,258 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:28,650 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:53:30,107 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:53:30,189 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:53:30,243 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:53:30,259 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:30,260 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:53:31,259 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:31,527 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:53:31,528 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:53:32,260 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:34,261 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:36,263 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:38,264 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:42,266 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:44,268 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:46,808 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:53:46,809 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:53:48,270 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:52,273 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:54,275 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:58,278 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:53:58,932 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:54:02,009 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:54:02,009 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:54:02,281 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:04,282 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:08,285 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:10,287 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:14,290 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:16,292 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:17,245 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:54:17,245 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:54:20,294 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:23,296 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:27,299 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:29,164 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:54:29,302 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:32,483 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:54:32,484 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:54:33,304 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:35,306 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:39,310 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:41,312 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:45,314 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:47,316 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:47,641 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:54:47,641 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:54:49,318 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:53,321 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:55,322 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:57,324 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:54:59,461 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:55:01,328 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:02,835 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:55:02,836 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:55:03,329 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:05,330 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:07,331 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:11,334 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:13,336 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:15,337 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:17,338 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:18,029 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:55:18,029 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:55:19,340 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:22,342 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:24,344 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:28,347 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:29,744 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:55:30,349 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:32,350 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:33,202 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:55:33,202 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:55:34,351 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:36,352 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:38,354 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:40,355 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:42,356 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:44,358 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:46,359 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:48,361 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:48,484 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:55:48,484 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:55:52,363 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:56,366 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:55:58,368 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:00,184 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:56:02,370 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:03,730 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:56:03,730 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:56:06,372 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:08,374 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:12,377 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:16,380 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:18,942 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:56:18,943 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:56:20,382 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:22,384 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:26,387 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:28,389 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:30,437 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:56:33,392 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:34,277 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:56:34,277 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:56:35,394 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:39,397 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:41,399 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:45,402 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:47,404 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:49,553 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:56:49,554 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:56:51,407 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:53,408 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:57,411 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:56:59,413 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:00,681 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:57:01,415 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:04,806 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:57:04,806 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:57:05,418 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:07,420 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:09,422 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:11,423 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:15,426 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:17,427 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:19,429 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:20,014 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:57:20,014 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:57:21,430 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:25,432 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:27,434 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:29,435 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:30,973 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:57:31,437 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:33,438 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:35,250 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:57:35,250 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:57:35,441 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:37,442 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:39,443 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:41,444 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:43,445 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:45,446 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:48,448 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:50,449 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:50,512 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:57:50,513 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:57:52,451 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:52,601 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:57:52,693 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:57:52,727 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:57:53,451 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:57:54,452 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:55,453 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:56,454 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:57:58,455 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:00,456 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:01,242 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:58:04,459 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:05,822 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:58:05,823 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:58:08,463 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:10,465 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:14,467 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:18,470 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:20,472 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:21,185 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:58:21,185 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:58:24,475 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:28,478 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:30,480 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:31,486 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:58:34,487 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:36,449 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:58:36,450 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:58:36,489 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:40,491 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:42,493 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:47,495 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:49,497 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:51,796 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:58:51,797 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:58:53,499 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:55,501 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:58:59,503 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:01,505 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:01,697 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:59:05,508 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:07,164 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:59:07,164 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:59:07,509 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:09,510 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:13,513 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:15,514 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:19,516 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:21,518 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:22,424 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:59:22,425 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:59:23,519 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:25,520 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:29,523 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:31,525 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:31,904 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:59:33,526 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:35,528 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:37,530 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:37,727 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:59:37,728 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:59:41,532 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:43,534 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:45,535 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:47,537 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:49,539 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:51,541 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:53,049 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:59:53,049 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:59:54,543 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:56,544 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:59:58,545 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:00,547 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:02,190 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:00:02,548 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:04,549 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:06,551 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:08,268 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:00:08,268 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:00:08,552 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:10,553 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:12,555 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:14,556 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:18,559 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:22,561 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:23,425 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:00:23,426 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:00:26,564 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:28,565 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:32,479 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:00:32,568 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:36,571 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:38,572 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:38,670 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:00:38,672 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:00:42,575 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:44,576 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:49,580 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:51,581 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:54,079 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:00:54,080 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:00:55,584 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:00:59,586 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:01,588 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:02,753 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:01:05,590 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:07,591 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:09,415 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:01:09,415 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:01:11,594 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:13,595 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:15,596 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:19,599 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:21,600 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:23,601 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:24,730 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:01:24,731 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:01:27,604 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:29,605 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:31,607 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:32,996 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:01:35,609 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:37,610 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:39,612 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:39,916 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:01:39,917 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:01:41,613 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:46,616 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:48,618 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:50,619 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:52,620 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:54,621 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:01:55,071 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:01:55,072 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:01:56,622 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:00,624 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:02,626 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:03,247 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:02:04,627 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:06,629 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:08,631 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:10,310 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:02:10,310 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:02:10,632 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:12,633 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:14,635 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:16,084 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 20:02:16,222 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 20:02:16,273 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 20:02:16,636 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:16,636 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 20:02:18,637 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:20,639 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:22,640 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:24,641 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:25,525 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:02:25,526 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:02:26,643 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:30,646 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:33,510 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:02:34,649 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:38,651 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:40,653 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:40,726 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:02:40,727 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:02:44,655 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:48,658 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:50,660 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:54,662 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:02:56,085 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:02:56,086 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:02:56,663 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:00,666 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:03,668 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:03,779 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:03:07,671 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:09,673 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:11,343 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:03:11,343 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:03:13,675 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:15,676 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:19,679 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:21,680 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:25,682 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:26,971 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:03:26,973 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:03:27,683 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:29,685 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:33,687 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:34,042 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:03:35,689 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:37,690 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:41,693 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:42,234 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:03:42,234 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:03:43,694 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:45,695 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:49,698 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:51,699 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:53,701 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:55,702 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:03:57,515 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:03:57,516 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:03:57,703 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:01,706 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:03,708 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:04,275 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:04:05,709 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:07,711 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:09,712 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:11,714 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:12,729 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:04:12,729 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:04:13,716 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:16,718 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:18,720 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:20,722 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:22,723 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:24,725 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:26,727 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:27,954 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:04:27,954 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:04:28,729 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:30,731 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:32,732 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:34,519 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:04:34,733 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:38,736 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:40,737 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:43,132 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:04:43,133 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:04:44,739 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:48,740 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:50,742 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:54,744 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:04:58,395 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:04:58,395 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:04:58,745 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:00,747 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:04,744 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:05:04,749 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:06,750 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:10,751 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:13,669 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:05:13,670 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:05:14,753 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:17,755 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:21,758 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:23,759 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:25,762 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:28,968 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:05:28,968 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:05:29,764 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:31,766 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:34,986 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:05:35,779 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:37,780 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:41,783 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:43,784 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:44,133 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:05:44,134 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:05:45,785 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:49,788 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:51,789 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:53,790 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:57,792 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:05:59,386 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:05:59,387 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:05:59,794 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:01,795 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:05,264 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:06:05,812 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:07,813 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:09,814 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:11,816 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:13,817 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:14,595 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:06:14,595 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:06:17,820 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:19,822 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:21,823 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:23,824 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:26,825 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:28,827 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:29,821 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:06:29,822 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:06:30,827 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:32,829 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:34,830 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:35,621 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:06:36,831 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:38,833 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:39,792 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 20:06:39,901 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 20:06:39,939 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 20:06:40,876 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:40,876 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 20:06:41,877 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:42,877 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:44,879 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:45,083 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:06:45,084 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:06:46,880 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:50,882 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:54,884 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:06:58,886 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:00,386 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:07:00,386 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:07:00,888 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:04,890 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:05,932 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:07:06,892 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:10,895 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:14,898 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:15,647 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:07:15,648 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:07:16,900 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:20,903 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:23,905 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:27,907 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:30,814 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:07:30,814 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:07:31,910 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:33,911 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:36,209 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:07:37,914 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:39,915 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:43,917 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:45,919 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:45,971 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:07:45,972 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:07:47,920 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:51,923 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:53,924 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:57,927 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:07:59,928 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:01,214 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:08:01,214 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:08:01,930 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:05,933 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:06,424 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:08:07,934 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:09,935 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:11,936 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:15,939 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:16,520 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:08:16,520 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:08:17,940 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:19,941 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:22,943 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:24,945 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:28,947 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:30,948 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:31,724 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:08:31,725 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:08:32,950 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:34,951 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:36,726 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:08:36,952 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:38,954 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:40,955 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:42,957 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:44,959 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:46,922 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:08:46,923 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:08:46,960 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:48,962 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:50,964 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:52,966 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:54,967 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:56,968 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:08:58,969 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:02,181 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:09:02,181 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:09:02,972 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:04,974 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:06,985 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:09:08,977 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:12,979 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:16,982 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:17,346 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:09:17,346 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:09:18,984 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:22,987 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:26,990 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:28,992 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:32,620 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:09:32,620 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:09:32,995 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:34,997 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:37,224 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:09:39,000 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:41,001 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:45,004 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:47,006 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:47,803 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:09:47,803 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:09:51,009 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:53,010 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:09:58,013 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:00,015 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:02,016 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:02,962 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:10:02,963 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:10:06,018 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:07,438 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:10:08,020 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:12,022 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:14,024 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:16,025 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:18,185 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:10:18,186 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:10:20,028 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:22,029 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:24,031 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:26,032 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:30,035 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:32,037 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:33,515 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:10:33,515 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:10:34,039 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:36,041 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:37,678 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:10:38,043 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:40,045 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:44,048 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:46,049 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:48,051 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:48,773 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:10:48,773 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:10:50,053 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:52,054 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:54,055 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:56,056 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:10:58,057 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:00,058 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:02,059 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:02,294 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 20:11:02,402 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 20:11:02,454 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 20:11:03,060 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 20:11:03,976 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:11:03,977 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:11:04,060 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:06,061 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:07,927 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:11:08,063 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:10,064 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:12,065 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:14,066 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:16,067 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:18,069 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:19,243 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:11:19,243 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:11:21,071 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:23,072 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:25,073 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:27,074 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:29,075 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:31,077 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:33,078 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:34,550 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:11:34,551 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:11:35,080 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:37,082 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:38,133 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:11:39,085 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:41,086 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:43,088 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:45,090 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:47,091 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:49,094 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:49,804 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:11:49,804 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:11:51,095 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:53,096 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:55,097 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:57,098 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:11:59,099 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:12:01,100 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:12:03,102 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:12:05,093 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:12:05,094 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:12:07,499 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 20:12:07,537 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 20:12:07,598 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 20:12:08,105 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:12:08,105 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 20:12:08,343 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:12:09,106 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:12:11,106 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:12:20,346 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:12:20,347 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:12:21,113 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 20:12:35,562 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:12:35,563 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:12:38,531 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:12:50,737 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:12:50,738 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:13:05,906 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:13:05,907 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:13:08,755 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 20:13:21,442 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:13:21,442 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:13:36,612 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 20:13:36,613 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 20:13:38,965 DEBUG SenderThread:13445 [sender.py:send():234] send: stats