diff --git "a/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" "b/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" --- "a/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" +++ "b/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" @@ -4031,3 +4031,939 @@ 2022-01-29 23:32:36,390 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status 2022-01-29 23:32:36,391 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status 2022-01-29 23:32:44,724 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:32:51,572 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:32:51,573 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:32:55,590 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:01,594 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:06,849 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:33:06,849 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:33:07,598 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:11,601 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:14,969 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:33:17,605 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:21,608 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:22,047 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:33:22,048 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:33:25,610 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:29,613 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:33,615 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:37,220 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:33:37,220 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:33:40,619 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:44,622 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:45,260 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:33:48,625 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:52,378 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:33:52,379 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:33:52,628 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:56,631 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:33:58,633 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:02,636 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:06,639 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:07,545 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:34:07,545 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:34:08,640 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:12,642 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:15,556 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:34:16,644 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:20,646 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:22,648 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:22,843 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:34:22,844 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:34:26,650 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:28,652 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:32,654 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:34,656 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:38,033 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:34:38,033 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:34:38,659 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:40,661 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:42,663 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:45,954 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:34:46,666 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:48,668 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:52,670 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:53,205 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:34:53,205 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:34:54,672 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:57,674 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:34:59,676 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:03,679 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:05,680 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:07,682 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:08,427 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:35:08,427 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:35:09,684 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:13,687 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:15,689 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:16,265 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:35:17,691 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:21,693 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:23,695 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:23,752 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:35:23,752 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:35:25,696 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:27,697 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:29,699 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:31,700 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:33,702 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:39,022 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:35:39,022 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:35:39,707 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:45,711 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:46,524 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:35:50,714 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:35:54,486 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:35:54,487 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:35:56,719 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:00,721 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:04,724 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:08,727 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:09,743 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:36:09,743 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:36:12,730 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:16,740 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:16,780 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:36:20,742 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:25,002 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:36:25,003 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:36:26,746 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:28,748 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:32,751 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:36,754 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:40,337 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:36:40,338 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:36:40,757 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:42,759 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:46,997 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:36:47,764 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:51,766 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:55,576 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:36:55,577 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:36:55,769 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:36:59,772 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:01,774 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:05,777 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:07,779 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:10,814 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:37:10,815 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:37:11,782 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:13,784 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:17,266 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:37:17,787 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:19,788 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:23,791 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:25,793 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:26,006 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:37:26,006 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:37:29,795 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:31,797 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:33,799 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:37,801 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:39,803 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:41,251 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:37:41,252 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:37:41,805 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:43,807 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:45,809 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:47,496 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:37:49,811 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:51,813 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:53,815 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:55,818 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:56,494 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:37:56,495 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:37:57,819 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:37:59,820 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:01,822 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:03,823 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:05,825 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:07,826 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:09,827 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:10,322 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:38:10,423 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:38:10,469 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:38:10,828 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:38:10,828 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:11,829 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:11,849 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:38:11,850 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:38:17,849 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:38:18,833 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:24,837 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:27,110 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:38:27,110 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:38:28,841 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:34,845 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:38,848 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:42,463 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:38:42,464 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:38:42,850 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:46,854 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:48,086 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:38:50,856 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:56,860 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:38:57,787 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:38:57,787 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:39:00,863 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:02,864 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:06,867 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:10,870 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:12,958 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:39:12,958 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:39:15,873 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:18,304 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:39:19,876 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:23,879 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:25,881 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:28,117 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:39:28,118 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:39:29,884 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:33,887 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:35,889 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:39,892 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:43,346 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:39:43,347 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:39:43,895 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:45,897 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:48,587 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:39:49,901 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:51,903 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:53,905 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:57,908 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:39:58,522 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:39:58,523 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:39:59,910 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:03,913 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:05,915 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:07,916 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:11,918 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:13,792 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:40:13,792 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:40:13,920 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:15,922 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:18,834 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:40:19,925 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:21,926 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:23,928 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:26,930 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:28,985 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:40:28,986 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:40:30,933 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:32,934 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:34,935 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:36,936 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:38,937 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:40,939 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:42,940 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:44,215 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:40:44,215 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:40:44,941 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:46,942 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:48,944 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:49,106 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:40:50,945 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:54,948 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:40:59,406 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:40:59,407 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:41:00,952 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:04,955 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:10,959 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:14,763 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:41:14,763 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:41:14,962 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:18,965 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:19,374 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:41:24,969 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:28,972 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:30,030 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:41:30,030 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:41:33,976 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:37,980 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:41,983 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:45,249 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:41:45,249 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:41:45,986 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:49,657 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:41:49,989 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:53,992 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:55,993 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:41:59,997 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:00,456 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:42:00,457 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:42:04,000 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:08,004 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:10,006 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:14,008 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:15,670 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:42:15,671 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:42:16,010 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:19,901 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:42:20,013 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:24,017 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:26,019 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:30,021 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:30,994 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:42:30,995 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:42:32,023 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:34,025 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:38,029 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:40,031 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:44,034 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:46,036 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:46,247 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:42:46,247 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:42:48,046 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:50,150 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:42:53,049 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:55,051 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:57,053 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:42:59,055 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:01,057 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:01,417 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:43:01,417 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:43:05,060 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:07,061 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:09,063 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:11,064 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:13,066 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:15,067 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:16,588 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:43:16,588 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:43:17,069 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:19,070 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:20,393 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:43:21,072 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:23,073 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:25,076 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:25,663 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:43:25,768 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:43:25,810 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:43:26,077 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:43:27,078 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:31,750 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:43:31,751 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:43:33,082 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:39,086 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:43,089 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:46,935 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:43:46,936 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:43:49,094 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:50,624 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:43:53,096 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:43:57,099 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:02,198 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:44:02,198 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:44:04,104 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:08,107 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:12,109 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:16,112 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:17,487 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:44:17,488 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:44:20,115 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:20,909 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:44:24,118 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:26,120 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:32,124 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:32,654 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:44:32,654 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:44:34,126 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:38,129 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:42,132 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:46,135 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:47,895 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:44:47,896 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:44:48,136 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:51,140 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:44:52,139 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:56,142 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:44:58,143 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:03,146 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:03,156 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:45:03,156 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:45:05,147 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:09,150 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:11,152 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:15,155 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:17,157 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:18,393 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:45:18,394 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:45:21,159 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:21,422 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:45:23,161 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:25,163 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:29,167 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:31,169 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:33,171 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:33,580 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:45:33,581 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:45:35,173 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:39,176 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:41,178 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:43,181 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:45,182 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:47,184 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:48,810 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:45:48,811 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:45:49,186 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:51,188 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:51,830 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:45:53,190 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:55,192 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:57,194 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:45:59,196 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:01,198 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:03,200 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:04,175 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:46:04,176 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:46:05,202 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:11,206 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:18,211 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:19,479 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:46:19,480 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:46:22,123 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:46:22,215 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:26,218 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:30,221 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:34,765 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:46:34,766 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:46:36,225 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:40,229 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:44,232 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:48,235 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:49,973 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:46:49,973 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:46:52,238 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:46:52,379 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:46:56,241 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:00,243 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:04,245 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:05,140 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:47:05,141 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:47:08,247 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:10,249 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:15,252 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:19,254 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:20,307 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:47:20,308 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:47:21,256 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:22,707 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:47:25,258 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:29,261 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:31,262 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:35,265 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:36,038 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:47:36,038 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:47:37,267 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:41,269 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:43,271 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:47,274 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:49,276 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:51,209 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:47:51,210 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:47:53,056 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:47:53,279 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:55,281 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:47:59,284 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:01,286 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:04,289 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:06,425 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:48:06,426 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:48:08,292 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:10,293 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:12,295 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:14,297 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:16,299 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:20,301 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:21,595 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:48:21,595 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:48:22,303 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:23,306 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:48:24,305 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:26,307 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:28,309 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:30,311 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:32,313 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:34,314 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:36,316 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:36,763 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:48:36,764 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:48:38,318 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:40,320 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:40,609 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:48:40,696 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:48:40,741 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:48:41,321 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:48:42,323 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:43,324 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:48,328 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:52,093 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:48:52,094 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:48:53,604 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:48:54,332 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:48:58,335 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:04,339 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:07,555 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:49:07,556 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:49:08,342 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:13,345 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:17,348 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:21,350 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:22,850 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:49:22,850 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:49:23,807 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:49:25,353 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:29,355 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:33,358 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:37,360 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:38,015 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:49:38,016 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:49:41,363 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:45,365 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:47,367 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:51,370 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:53,184 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:49:53,184 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:49:54,042 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:49:55,372 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:49:59,375 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:01,377 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:05,380 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:08,382 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:08,394 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:50:08,394 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:50:12,385 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:14,387 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:18,390 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:20,392 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:23,571 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:50:23,571 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:50:24,313 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:50:24,394 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:26,395 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:30,397 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:32,398 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:34,399 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:38,402 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:38,824 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:50:38,824 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:50:40,404 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:42,405 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:46,408 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:48,410 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:50,411 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:52,412 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:54,148 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:50:54,149 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:50:54,414 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:50:54,569 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:50:58,416 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:00,417 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:02,418 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:04,419 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:06,421 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:08,422 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:09,394 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:51:09,395 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:51:10,423 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:12,425 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:14,426 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:16,427 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:19,429 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:24,642 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:51:24,642 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:51:24,808 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:51:25,433 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:31,437 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:35,440 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:39,443 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:39,999 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:51:40,000 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:51:45,446 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:49,449 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:53,451 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:51:55,054 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:51:55,245 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:51:55,245 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:51:57,453 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:02,456 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:06,459 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:10,409 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:52:10,410 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:52:10,461 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:14,462 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:18,464 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:22,466 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:25,326 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:52:25,573 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:52:25,573 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:52:26,469 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:28,470 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:32,472 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:36,475 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:38,477 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:40,828 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:52:40,829 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:52:42,480 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:44,482 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:48,485 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:52,488 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:54,490 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:52:55,610 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:52:56,113 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:52:56,113 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:52:58,493 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:00,495 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:02,497 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:06,500 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:08,502 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:11,488 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:53:11,489 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:53:13,504 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:15,506 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:17,507 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:21,509 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:23,510 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:25,512 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:25,838 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:53:26,774 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:53:26,774 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:53:27,513 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:29,514 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:33,517 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:35,519 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:37,521 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:39,523 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:41,525 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:42,031 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:53:42,031 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:53:43,527 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:45,529 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:47,531 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:49,532 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:51,533 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:53,535 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:54,069 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:53:54,158 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:53:54,189 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:53:54,536 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:53:55,536 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:56,210 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:53:56,537 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:53:57,249 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:53:57,250 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:54:01,540 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:07,544 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:11,547 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:12,531 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:54:12,531 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:54:17,551 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:22,554 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:26,446 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:54:26,557 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:27,796 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:54:27,796 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:54:30,560 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:34,563 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:38,565 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:42,567 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:43,115 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:54:43,116 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:54:46,570 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:50,572 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:54,575 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:54:56,660 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:54:58,348 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:54:58,348 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:54:58,578 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:02,581 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:04,583 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:09,586 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:11,587 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:13,514 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:55:13,514 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:55:15,590 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:19,593 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:21,595 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:25,597 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:26,925 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:55:27,599 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:28,682 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:55:28,683 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:55:31,601 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:33,602 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:37,605 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:39,607 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:43,609 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:43,929 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:55:43,929 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:55:45,615 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:47,616 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:51,619 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:53,620 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:55,622 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:55:57,148 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:55:59,166 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:55:59,166 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:55:59,624 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:01,626 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:03,628 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:05,629 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:07,631 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:11,633 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:13,635 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:14,402 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:56:14,402 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:56:15,637 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:17,639 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:19,641 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:21,643 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:23,645 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:25,646 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:27,473 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:56:28,649 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:29,631 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:56:29,631 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:56:30,651 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:32,653 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:38,658 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:42,661 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:44,929 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:56:44,930 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:56:48,665 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:52,668 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:56,671 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:56:57,770 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:57:00,099 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:57:00,100 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:57:02,675 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:06,678 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:11,682 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:15,303 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:57:15,304 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:57:15,685 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:19,688 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:23,691 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:27,694 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:28,007 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:57:30,578 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:57:30,578 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:57:31,697 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:35,699 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:37,701 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:41,703 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:45,706 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:45,807 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:57:45,807 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:57:49,707 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:51,708 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:55,711 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:57,713 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:57:58,275 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:58:01,055 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:58:01,056 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:58:01,716 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:03,718 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:07,721 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:11,724 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:14,726 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:16,711 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:58:16,712 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:58:18,728 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:20,730 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:24,732 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:26,734 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:28,490 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:58:28,736 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:31,983 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:58:31,983 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:58:32,739 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:34,741 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:36,743 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:40,746 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:42,748 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:44,750 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:46,752 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:47,145 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:58:47,146 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:58:50,755 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:52,756 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:54,758 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:56,760 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:58,762 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:58:58,764 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:59:00,763 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:02,310 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:59:02,311 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:59:02,766 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:04,768 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:06,770 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:08,772 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:09,619 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:59:09,713 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:59:09,756 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:59:09,773 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:59:10,774 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:14,778 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:16,780 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:17,515 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:59:17,516 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:59:20,783 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:22,785 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:26,788 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:28,790 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:29,016 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:59:32,713 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:59:32,713 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:59:32,793 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:34,794 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:36,796 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:40,798 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:42,800 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:45,802 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:48,002 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:59:48,002 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:59:49,805 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:51,807 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:53,809 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:57,812 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:59:59,267 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:59:59,813 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:01,815 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:03,244 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:00:03,245 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:00:05,819 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:07,820 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:11,823 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:13,825 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:15,827 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:18,438 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:00:18,439 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:00:19,830 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:21,832 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:23,834 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:27,837 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:29,527 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:00:29,839 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:31,841 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:33,695 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:00:33,696 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:00:35,844 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:37,846 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:39,848 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:43,851 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:45,853 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:47,855 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:48,966 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:00:48,966 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:00:51,858 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:53,858 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:55,860 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:00:59,785 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:00:59,863 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:02,866 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:04,241 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:01:04,242 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:01:06,869 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:08,871 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:10,873 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:14,876 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:16,878 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:19,468 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:01:19,468 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:01:20,880 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:22,882 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:24,884 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:28,888 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:30,002 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:01:30,890 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:32,892 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:34,713 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:01:34,714 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:01:34,894 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:38,897 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:40,899 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:42,901 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:46,904 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:48,906 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:49,960 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:01:49,960 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:01:52,909 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:54,911 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:01:56,913 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:00,253 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:02:00,916 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:02,918 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:04,920 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:05,221 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:02:05,222 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:02:08,923 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:11,925 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:13,926 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:17,929 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:19,930 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:20,436 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:02:20,436 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:02:23,933 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:25,935 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:27,937 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:30,484 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:02:31,941 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:33,943 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:35,704 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:02:35,704 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:02:37,946 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:39,948 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:41,950 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:45,953 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:47,955 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:50,977 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:02:50,978 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:02:51,958 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:53,960 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:55,962 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:02:59,965 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:00,726 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:03:01,967 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:05,970 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:06,200 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:03:06,201 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:03:07,971 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:09,972 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:11,974 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:16,977 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:18,979 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:21,423 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:03:21,424 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:03:22,982 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:03:30,996 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:03:36,607 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:03:36,607 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:03:51,822 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:03:51,823 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:04:01,198 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:04:07,036 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:04:07,037 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:04:10,881 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-30 00:04:10,915 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-30 00:04:10,962 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-30 00:04:11,011 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-30 00:04:13,013 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:04:15,015 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:04:22,234 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:04:22,235 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:04:25,021 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:04:31,423 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:04:37,405 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:04:37,405 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:04:52,571 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:04:52,571 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:05:01,635 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:05:07,742 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:05:07,742 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:05:22,917 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:05:22,918 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-30 00:05:23,056 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-30 00:05:31,874 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-30 00:05:38,075 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 00:05:38,076 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status