diff --git "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" --- "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" +++ "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" @@ -13644,3 +13644,621 @@ 2022-02-02 04:16:00,505 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status 2022-02-02 04:16:15,675 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status 2022-02-02 04:16:15,676 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:16:24,311 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:26,312 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:28,314 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:29,796 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:16:30,316 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:30,924 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:16:30,925 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:16:32,318 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:34,320 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:36,322 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:38,324 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:40,326 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:42,328 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:44,330 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:46,147 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:16:46,147 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:16:46,332 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:48,334 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:50,336 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:52,338 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:54,340 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:56,342 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:16:58,343 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:00,004 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:17:01,310 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:17:01,310 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:17:02,347 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:04,349 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:06,350 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:08,353 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:10,355 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:12,357 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:14,359 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:16,360 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:16,557 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:17:16,558 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:17:18,363 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:20,364 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:22,366 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:24,368 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:26,370 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:28,372 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:30,253 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:17:30,374 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:31,822 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:17:31,822 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:17:32,376 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:34,378 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:36,380 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:38,381 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:40,383 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:42,384 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:44,385 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:46,386 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:47,055 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:17:47,055 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:17:48,388 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:50,389 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:52,390 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:54,391 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:56,393 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:17:58,394 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:00,395 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:00,523 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:18:02,233 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:18:02,234 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:18:02,397 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:04,398 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:06,399 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:08,400 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:10,402 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:13,404 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:15,406 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:17,408 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:17,441 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:18:17,442 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:18:19,411 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:21,385 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:18:21,439 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:21,463 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:18:21,492 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:18:22,440 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:18:23,441 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:24,443 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:25,444 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:27,446 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:29,448 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:30,768 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:18:31,449 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:32,709 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:18:32,710 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:18:33,450 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:37,453 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:39,454 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:41,456 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:43,457 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:45,459 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:47,460 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:48,077 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:18:48,078 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:18:49,461 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:51,463 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:53,465 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:55,467 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:57,469 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:18:59,471 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:01,133 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:19:01,473 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:03,347 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:19:03,347 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:19:03,474 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:05,476 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:07,477 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:09,478 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:11,479 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:13,481 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:15,482 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:17,483 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:18,529 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:19:18,530 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:19:19,485 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:21,486 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:23,487 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:25,488 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:27,489 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:29,491 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:31,388 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:19:31,492 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:33,494 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:33,697 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:19:33,697 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:19:37,496 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:39,498 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:41,499 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:43,501 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:45,503 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:47,505 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:48,867 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:19:48,868 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:19:49,506 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:51,508 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:53,510 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:55,511 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:19:58,513 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:00,515 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:01,695 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:20:02,519 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:04,076 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:20:04,076 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:20:04,520 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:06,521 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:08,522 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:10,524 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:12,525 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:14,526 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:16,527 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:18,529 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:19,255 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:20:19,256 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:20:20,531 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:22,533 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:22,677 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:20:22,738 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:20:22,768 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:20:23,534 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:20:24,535 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:26,537 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:28,539 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:30,541 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:31,912 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:20:32,543 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:34,430 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:20:34,431 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:20:36,546 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:38,548 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:40,550 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:42,552 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:44,553 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:46,555 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:48,557 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:49,629 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:20:49,630 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:20:50,559 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:52,561 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:54,562 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:56,564 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:20:58,566 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:00,567 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:02,240 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:21:02,568 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:04,570 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:04,875 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:21:04,876 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:21:06,571 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:08,572 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:10,574 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:13,576 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:15,577 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:17,579 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:19,580 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:20,079 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:21:20,079 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:21:21,581 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:23,583 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:25,584 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:27,585 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:29,587 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:31,588 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:32,500 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:21:33,590 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:35,281 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:21:35,282 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:21:35,591 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:37,593 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:39,595 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:41,597 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:43,599 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:45,601 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:47,603 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:49,605 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:50,437 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:21:50,438 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:21:51,607 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:53,609 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:55,611 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:57,613 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:21:59,615 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:01,617 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:02,753 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:22:03,619 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:05,621 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:05,709 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:22:05,710 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:22:07,623 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:09,625 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:11,627 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:13,629 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:15,631 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:17,633 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:19,636 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:20,928 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:22:20,929 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:22:21,234 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:22:21,329 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:22:21,375 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:22:21,639 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:21,639 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:22:22,640 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:24,642 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:26,644 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:28,645 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:30,648 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:32,650 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:33,000 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:22:36,238 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:22:36,239 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:22:36,653 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:38,654 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:40,655 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:42,657 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:44,658 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:46,659 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:48,661 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:50,662 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:51,554 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:22:51,555 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:22:52,663 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:54,665 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:56,666 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:22:58,668 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:00,669 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:02,670 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:03,239 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:23:04,671 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:06,673 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:06,792 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:23:06,792 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:23:08,674 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:10,675 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:12,677 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:14,678 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:16,679 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:18,681 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:20,682 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:21,961 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:23:21,962 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:23:22,683 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:24,685 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:26,686 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:28,687 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:30,690 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:32,692 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:33,510 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:23:34,693 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:36,694 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:37,134 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:23:37,135 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:23:38,696 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:40,698 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:42,699 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:44,700 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:46,702 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:48,703 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:50,704 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:52,375 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:23:52,375 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:23:52,705 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:54,706 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:57,708 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:23:59,709 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:01,711 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:03,712 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:03,779 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:24:05,713 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:07,545 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:24:07,545 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:24:07,715 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:09,716 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:11,717 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:13,719 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:15,720 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:17,721 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:19,722 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:20,997 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:24:21,105 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:24:21,141 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:24:21,723 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:21,724 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:24:22,763 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:24:22,765 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:24:23,725 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:25,726 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:27,727 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:29,728 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:31,730 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:34,044 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:24:35,732 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:37,733 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:37,995 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:24:37,996 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:24:39,734 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:41,735 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:43,737 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:45,738 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:47,739 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:49,741 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:51,742 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:53,355 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:24:53,356 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:24:53,743 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:55,745 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:57,746 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:24:59,748 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:01,749 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:03,750 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:04,329 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:25:05,751 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:07,754 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:08,668 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:25:08,668 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:25:09,755 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:11,756 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:13,758 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:15,759 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:17,760 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:19,761 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:21,762 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:23,763 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:23,863 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:25:23,863 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:25:25,765 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:27,766 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:29,768 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:31,769 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:33,771 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:34,578 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:25:35,772 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:37,774 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:39,025 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:25:39,025 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:25:42,776 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:44,778 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:46,779 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:48,781 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:50,782 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:52,783 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:54,204 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:25:54,218 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:25:54,785 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:56,786 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:25:58,787 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:00,788 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:02,789 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:04,791 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:04,874 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:26:06,792 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:08,793 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:09,484 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:26:09,485 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:26:10,795 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:12,796 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:14,797 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:16,799 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:18,800 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:20,801 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:21,545 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:26:21,646 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:26:21,683 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:26:21,801 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:26:22,802 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:23,803 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:24,717 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:26:24,718 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:26:24,804 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:26,805 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:28,806 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:30,808 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:32,809 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:34,810 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:35,153 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:26:36,812 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:38,813 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:39,920 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:26:39,921 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:26:40,814 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:42,815 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:44,816 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:46,818 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:48,819 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:50,820 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:52,822 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:55,123 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:26:55,123 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:26:55,824 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:57,825 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:26:59,826 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:01,827 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:03,829 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:05,385 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:27:05,830 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:07,831 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:09,833 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:10,327 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:27:10,327 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:27:11,834 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:13,836 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:15,837 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:17,838 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:19,839 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:21,841 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:23,842 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:25,736 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:27:25,736 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:27:25,844 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:27,845 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:29,846 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:31,847 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:33,848 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:35,595 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:27:35,849 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:37,851 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:39,852 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:41,023 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:27:41,024 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:27:41,853 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:43,855 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:45,856 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:47,857 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:49,858 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:51,860 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:53,861 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:55,863 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:56,253 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:27:56,254 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:27:57,864 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:27:59,865 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:01,866 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:04,869 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:05,841 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:28:06,870 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:08,871 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:10,873 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:11,529 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:28:11,530 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:28:12,874 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:14,875 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:16,876 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:18,878 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:20,879 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:22,880 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:24,882 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:26,701 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:28:26,701 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:28:26,883 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:28,884 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:30,885 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:32,887 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:34,888 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:36,110 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:28:36,890 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:38,891 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:40,892 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:41,870 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:28:41,870 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:28:42,893 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:44,895 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:46,897 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:48,899 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:50,900 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:52,903 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:54,905 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:56,907 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:28:57,029 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:28:57,029 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:28:58,908 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:00,910 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:02,912 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:04,914 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:06,332 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:29:06,916 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:08,918 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:10,919 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:12,199 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:29:12,199 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:29:12,921 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:14,923 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:16,924 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:19,926 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:21,928 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:23,929 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:25,930 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:27,430 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:29:27,431 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:29:27,931 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:29,932 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:31,934 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:33,935 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:35,937 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:36,636 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:29:37,939 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:39,941 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:41,943 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:42,600 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:29:42,600 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:29:43,945 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:45,947 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:47,948 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:49,948 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:51,950 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:53,951 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:55,953 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:57,865 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:29:57,865 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:29:57,954 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:29:59,955 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:01,956 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:03,957 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:05,957 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:06,951 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:30:07,958 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:09,959 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:11,960 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:13,138 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:30:13,139 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:30:13,961 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:15,963 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:17,964 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:19,966 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:21,966 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:23,968 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:25,969 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:27,970 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:28,475 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:30:28,476 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:30:29,971 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:32,973 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:34,974 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:36,976 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:37,253 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:30:38,977 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:40,979 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:42,981 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:43,675 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:30:43,675 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:30:44,982 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:46,983 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:48,985 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:50,986 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:30:58,872 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:30:58,872 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:31:07,486 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:31:14,095 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:31:14,096 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:31:29,347 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:31:29,348 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:31:37,778 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:31:44,560 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:31:44,561 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:31:49,250 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:31:49,280 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:31:49,326 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:31:50,021 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:31:51,022 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:31:53,024 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:31:59,754 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:31:59,756 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:32:01,029 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:32:07,994 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:32:14,924 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:32:14,925 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:32:30,102 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:32:30,102 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:32:38,269 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:32:45,275 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:32:45,275 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:33:00,447 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:33:00,447 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:33:08,496 DEBUG SenderThread:38198 [sender.py:send():234] send: stats