diff --git "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" --- "a/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" +++ "b/wandb/run-20220201_223624-2b1hcyq3/logs/debug-internal.log" @@ -13030,3 +13030,617 @@ 2022-02-02 03:59:15,872 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status 2022-02-02 03:59:15,873 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status 2022-02-02 03:59:20,952 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:59:30,520 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:31,121 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:59:31,122 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:59:32,521 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:34,522 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:36,524 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:38,525 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:40,526 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:42,528 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:44,529 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:46,344 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 03:59:46,345 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 03:59:46,530 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:48,532 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:50,533 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:51,246 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 03:59:52,535 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:54,536 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:56,537 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 03:59:58,538 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:00,540 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:01,516 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:00:01,516 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:00:02,541 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:04,542 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:06,544 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:08,545 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:12,548 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:14,549 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:16,763 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:00:16,763 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:00:17,551 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:19,552 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:21,445 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:00:21,554 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:23,555 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:25,556 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:27,557 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:29,559 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:31,560 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:32,077 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:00:32,078 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:00:33,561 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:35,563 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:37,565 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:39,567 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:41,569 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:43,571 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:45,572 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:47,238 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:00:47,238 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:00:47,574 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:49,576 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:51,578 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:51,659 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:00:53,580 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:55,582 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:57,583 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:00:59,585 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:01,587 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:02,494 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:01:02,494 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:01:03,589 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:05,591 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:09,594 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:11,596 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:13,598 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:15,600 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:17,602 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:17,758 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:01:17,758 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:01:19,604 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:21,606 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:21,861 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:01:23,608 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:25,609 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:27,611 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:28,086 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:01:28,177 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:01:28,220 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:01:28,612 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:01:29,613 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:31,614 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:32,928 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:01:32,930 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:01:33,615 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:35,617 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:37,618 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:40,620 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:42,621 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:44,622 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:46,624 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:48,133 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:01:48,134 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:01:48,625 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:50,627 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:52,092 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:01:52,628 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:54,630 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:56,631 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:01:58,632 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:00,634 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:02,636 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:03,308 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:02:03,308 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:02:06,639 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:08,641 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:10,643 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:12,645 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:14,646 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:16,648 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:18,628 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:02:18,628 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:02:18,650 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:20,652 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:22,318 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:02:22,654 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:24,656 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:26,658 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:28,660 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:30,662 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:32,664 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:33,892 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:02:33,893 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:02:34,666 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:36,668 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:38,670 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:40,671 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:42,673 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:44,675 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:46,677 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:48,679 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:49,059 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:02:49,059 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:02:50,680 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:52,616 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:02:52,682 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:54,684 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:56,686 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:02:58,688 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:00,690 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:02,692 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:04,334 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:03:04,334 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:03:04,693 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:08,695 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:10,697 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:12,698 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:14,700 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:16,700 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:18,702 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:19,574 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:03:19,575 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:03:21,703 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:22,851 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:03:23,705 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:25,706 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:27,708 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:27,865 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:03:27,948 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:03:27,997 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:03:28,709 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:03:29,710 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:30,711 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:31,712 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:33,714 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:34,794 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:03:34,795 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:03:35,716 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:37,718 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:39,719 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:41,721 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:43,723 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:45,725 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:47,727 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:49,729 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:50,036 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:03:50,037 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:03:51,730 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:53,087 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:03:53,732 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:55,733 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:57,734 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:03:59,735 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:01,736 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:03,739 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:05,308 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:04:05,309 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:04:05,740 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:07,742 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:09,743 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:11,745 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:13,746 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:15,747 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:17,749 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:19,750 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:20,559 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:04:20,560 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:04:21,751 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:23,302 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:04:24,753 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:26,754 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:28,756 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:30,757 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:32,759 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:34,760 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:35,804 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:04:35,805 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:04:36,762 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:38,763 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:40,764 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:42,766 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:44,767 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:46,769 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:48,770 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:50,772 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:51,014 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:04:51,014 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:04:52,773 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:53,576 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:04:54,775 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:56,777 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:04:58,779 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:00,780 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:02,782 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:04,783 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:06,298 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:05:06,299 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:05:08,786 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:10,787 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:12,789 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:14,790 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:16,791 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:18,793 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:20,794 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:21,592 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:05:21,592 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:05:22,795 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:23,816 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:05:24,796 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:26,797 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:27,034 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:05:27,150 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:05:27,186 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:05:27,798 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:05:28,798 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:30,800 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:33,802 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:35,803 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:36,760 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:05:36,761 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:05:37,804 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:39,806 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:41,807 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:43,809 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:45,811 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:47,813 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:49,814 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:51,816 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:51,913 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:05:51,913 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:05:53,818 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:54,073 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:05:55,820 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:57,822 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:05:59,823 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:01,825 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:05,828 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:07,076 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:06:07,077 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:06:07,830 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:09,832 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:11,834 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:13,836 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:15,837 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:17,839 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:19,840 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:21,844 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:22,251 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:06:22,252 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:06:23,846 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:24,396 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:06:25,848 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:27,850 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:29,852 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:31,854 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:33,857 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:35,858 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:37,468 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:06:37,469 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:06:37,860 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:39,861 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:41,863 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:43,864 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:45,865 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:47,866 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:49,868 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:51,869 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:52,631 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:06:52,632 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:06:53,870 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:54,653 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:06:55,871 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:06:57,873 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:00,875 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:04,879 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:06,881 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:07,885 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:07:07,886 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:07:08,882 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:10,884 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:12,886 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:14,888 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:16,890 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:18,891 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:20,893 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:22,895 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:23,062 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:07:23,062 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:07:24,897 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:24,972 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:07:25,534 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:07:25,642 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:07:25,674 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:07:25,898 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:07:26,900 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:28,902 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:30,904 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:32,906 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:34,907 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:36,909 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:38,296 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:07:38,297 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:07:38,911 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:40,913 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:42,914 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:44,916 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:46,918 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:48,920 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:50,922 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:52,924 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:53,558 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:07:53,558 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:07:54,925 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:55,303 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:07:56,927 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:07:58,929 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:00,931 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:04,934 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:06,937 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:08,723 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:08:08,724 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:08:08,939 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:10,940 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:12,943 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:14,944 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:16,946 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:18,948 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:20,950 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:22,952 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:23,923 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:08:23,924 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:08:24,954 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:25,577 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:08:26,955 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:28,957 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:30,959 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:32,961 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:34,963 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:36,965 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:38,966 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:39,102 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:08:39,102 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:08:40,967 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:42,969 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:44,970 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:46,971 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:48,972 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:50,974 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:52,975 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:54,364 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:08:54,365 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:08:54,977 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:55,829 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:08:56,978 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:08:59,980 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:01,981 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:05,984 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:07,985 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:09,528 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:09:09,529 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:09:10,008 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:12,010 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:14,011 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:16,012 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:18,013 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:20,014 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:22,015 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:24,017 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:24,724 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:09:24,725 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:09:26,018 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:26,208 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:09:27,323 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:09:27,381 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:09:27,409 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:09:28,019 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:28,020 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:09:30,021 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:32,022 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:34,023 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:36,025 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:38,026 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:39,889 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:09:39,891 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:09:40,027 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:42,029 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:44,030 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:46,032 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:48,033 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:50,034 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:52,036 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:54,037 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:55,060 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:09:55,061 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:09:56,038 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:09:56,477 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:09:58,039 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:00,040 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:02,042 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:04,043 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:07,044 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:09,046 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:10,270 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:10:10,271 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:10:11,047 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:13,048 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:15,049 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:17,050 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:19,051 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:21,052 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:23,053 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:25,054 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:25,579 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:10:25,580 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:10:26,771 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:10:27,056 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:29,057 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:31,058 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:33,059 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:35,060 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:37,061 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:39,063 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:40,887 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:10:40,887 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:10:41,064 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:43,065 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:45,067 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:47,068 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:49,069 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:51,070 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:53,071 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:55,072 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:56,057 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:10:56,058 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:10:57,074 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:10:57,084 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:10:59,076 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:01,077 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:03,078 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:05,079 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:07,081 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:09,082 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:11,084 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:11,265 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:11:11,265 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:11:13,085 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:15,086 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:17,087 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:19,089 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:21,090 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:23,091 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:25,092 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:26,509 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:11:26,510 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:11:27,093 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:27,306 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:11:29,094 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:32,096 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:34,098 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:36,099 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:38,100 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:40,102 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:41,786 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:11:41,786 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:11:42,103 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:44,104 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:46,105 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:48,107 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:50,108 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:52,109 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:54,110 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:56,112 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:11:57,022 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:11:57,022 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:11:57,565 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:11:58,113 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:00,114 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:02,115 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:04,117 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:06,121 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:08,122 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:10,123 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:12,125 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:12,284 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:12:12,284 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:12:14,125 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:16,127 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:18,128 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:20,129 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:22,130 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:24,132 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:26,133 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:27,472 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:12:27,472 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:12:27,840 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:12:28,135 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:30,136 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:32,138 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:34,140 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:36,142 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:38,144 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:40,146 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:42,148 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:42,635 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:12:42,636 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:12:44,150 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:46,152 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:48,154 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:50,155 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:52,157 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:55,160 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:57,161 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:12:57,862 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:12:57,862 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:12:58,105 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:12:59,163 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:01,165 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:03,167 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:05,169 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:07,171 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:09,172 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:11,174 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:13,161 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:13:13,162 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:13:13,176 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:15,178 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:17,179 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:19,181 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:21,182 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:23,183 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:25,184 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:27,186 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:28,325 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:13:28,448 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:13:28,448 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:13:29,187 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:31,188 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:33,190 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:35,191 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:37,192 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:39,194 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:41,195 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:43,196 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:43,674 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:13:43,674 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:13:45,197 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:47,198 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:49,200 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:51,201 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:53,203 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:55,204 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:57,206 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:13:58,635 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:13:58,874 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:13:58,875 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:14:14,113 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:14:14,113 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:14:28,860 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:14:29,333 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:14:29,333 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:14:44,552 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:14:44,553 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:14:55,189 DEBUG SenderThread:38198 [sender.py:send():234] send: history +2022-02-02 04:14:55,224 DEBUG SenderThread:38198 [sender.py:send():234] send: summary +2022-02-02 04:14:55,271 INFO SenderThread:38198 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-02-02 04:14:56,247 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/wandb-summary.json +2022-02-02 04:14:57,248 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:14:59,106 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:14:59,749 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:14:59,750 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:15:07,254 INFO Thread-8 :38198 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-pt/wandb/run-20220201_223624-2b1hcyq3/files/output.log +2022-02-02 04:15:14,919 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:15:14,919 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:15:29,370 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:15:30,097 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:15:30,097 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:15:45,267 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:15:45,268 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:15:59,569 DEBUG SenderThread:38198 [sender.py:send():234] send: stats +2022-02-02 04:16:00,504 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:16:00,505 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status +2022-02-02 04:16:15,675 DEBUG HandlerThread:38198 [handler.py:handle_request():130] handle_request: stop_status +2022-02-02 04:16:15,676 DEBUG SenderThread:38198 [sender.py:send_request():248] send_request: stop_status