diff --git "a/wandb/run-20220130_191954-1mtmnz5y/logs/debug-internal.log" "b/wandb/run-20220130_191954-1mtmnz5y/logs/debug-internal.log" --- "a/wandb/run-20220130_191954-1mtmnz5y/logs/debug-internal.log" +++ "b/wandb/run-20220130_191954-1mtmnz5y/logs/debug-internal.log" @@ -4016,3 +4016,934 @@ 2022-01-30 21:01:01,742 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status 2022-01-30 21:01:01,814 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log 2022-01-30 21:01:15,675 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:01:16,910 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:01:16,911 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:01:20,826 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:01:28,833 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:01:32,083 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:01:32,083 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:01:34,837 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:01:40,841 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:01:45,844 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:01:45,918 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:01:47,256 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:01:47,256 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:01:51,847 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:01:55,849 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:01:59,852 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:02,423 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:02:02,424 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:02:05,856 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:09,858 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:13,861 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:16,184 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:02:17,751 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:02:17,752 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:02:17,864 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:21,867 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:25,870 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:29,873 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:33,015 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:02:33,016 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:02:33,875 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:38,878 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:42,881 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:46,395 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:02:46,884 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:48,395 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:02:48,395 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:02:50,887 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:52,889 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:02:56,892 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:00,895 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:02,896 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:03,687 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:03:03,687 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:03:06,899 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:10,902 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:12,904 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:16,624 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:03:16,908 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:18,909 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:18,917 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:03:18,917 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:03:22,911 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:24,913 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:28,916 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:31,919 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:34,235 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:03:34,235 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:03:35,922 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:37,923 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:39,926 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:43,928 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:45,930 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:46,902 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:03:47,932 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:49,511 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:03:49,512 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:03:49,934 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:53,937 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:55,940 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:57,942 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:03:59,943 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:01,945 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:03,947 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:04,869 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:04:04,869 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:04:05,949 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:07,951 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:09,953 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:11,955 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:13,957 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:17,194 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:04:19,966 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:20,246 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:04:20,247 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:04:25,970 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:32,975 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:35,696 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:04:35,696 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:04:36,978 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:42,982 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:46,985 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:47,441 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:04:50,988 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:51,028 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:04:51,029 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:04:54,990 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:04:58,992 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:04,996 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:06,325 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:05:06,326 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:05:08,999 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:13,001 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:17,006 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:17,685 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:05:21,009 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:21,496 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:05:21,496 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:05:25,012 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:29,014 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:31,019 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:36,023 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:36,663 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:05:36,664 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:05:40,026 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:44,028 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:46,030 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:47,938 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:05:50,032 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:51,953 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:05:51,954 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:05:54,035 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:05:56,038 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:00,041 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:02,042 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:06,045 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:07,197 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:06:07,197 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:06:08,046 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:12,049 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:14,050 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:18,053 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:18,236 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:06:20,055 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:22,499 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:06:22,499 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:06:24,058 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:26,060 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:28,061 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:31,064 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:35,067 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:37,068 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:37,794 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:06:37,794 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:06:39,071 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:41,072 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:43,075 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:45,077 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:48,489 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:06:49,079 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:51,081 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:53,055 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:06:53,055 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:06:53,083 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:55,085 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:57,087 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:59,089 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:06:59,376 DEBUG SenderThread:55777 [sender.py:send():234] send: history +2022-01-30 21:06:59,474 DEBUG SenderThread:55777 [sender.py:send():234] send: summary +2022-01-30 21:06:59,520 INFO SenderThread:55777 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-30 21:07:00,090 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/wandb-summary.json +2022-01-30 21:07:01,091 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:08,434 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:07:08,435 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:07:09,097 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:15,101 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:18,780 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:07:20,104 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:23,920 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:07:23,921 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:07:26,109 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:30,112 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:36,116 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:39,328 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:07:39,328 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:07:40,119 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:44,122 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:48,126 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:49,027 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:07:52,129 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:07:54,618 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:07:54,618 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:07:57,132 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:01,135 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:05,138 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:09,141 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:09,784 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:08:09,785 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:08:13,144 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:17,147 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:19,250 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:08:21,150 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:23,152 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:25,359 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:08:25,359 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:08:27,155 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:31,158 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:33,160 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:37,163 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:40,561 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:08:40,562 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:08:41,167 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:43,168 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:48,172 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:49,495 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:08:50,174 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:54,177 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:08:55,764 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:08:55,765 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:08:56,179 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:00,182 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:02,184 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:04,186 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:08,189 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:10,191 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:10,934 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:09:10,935 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:09:12,192 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:16,196 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:18,197 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:19,802 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:09:20,199 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:22,200 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:26,202 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:26,276 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:09:26,277 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:09:28,203 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:30,205 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:32,206 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:34,208 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:36,210 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:38,212 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:40,214 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:41,561 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:09:41,561 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:09:42,216 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:44,217 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:46,219 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:50,075 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:09:54,225 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:09:56,731 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:09:56,731 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:10:00,229 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:07,233 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:11,235 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:11,889 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:10:11,890 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:10:17,238 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:20,375 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:10:21,242 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:25,245 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:27,066 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:10:27,067 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:10:29,248 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:35,252 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:39,255 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:42,492 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:10:42,492 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:10:43,258 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:47,261 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:50,616 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:10:51,264 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:56,267 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:10:57,881 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:10:57,882 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:10:58,269 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:02,272 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:06,275 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:10,278 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:13,267 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:11:13,268 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:11:14,282 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:16,284 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:20,287 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:20,830 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:11:24,290 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:26,292 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:28,565 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:11:28,565 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:11:30,294 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:34,298 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:36,300 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:40,302 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:42,304 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:43,734 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:11:43,734 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:11:46,307 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:49,309 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:51,063 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:11:51,311 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:55,314 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:57,316 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:11:58,978 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:11:58,979 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:12:01,319 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:03,321 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:05,323 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:07,325 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:11,328 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:13,330 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:14,318 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:12:14,319 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:12:15,332 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:17,333 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:19,335 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:21,302 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:12:21,337 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:23,339 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:25,341 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:27,343 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:29,344 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:29,588 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:12:29,588 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:12:31,346 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:33,347 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:33,707 DEBUG SenderThread:55777 [sender.py:send():234] send: history +2022-01-30 21:12:33,819 DEBUG SenderThread:55777 [sender.py:send():234] send: summary +2022-01-30 21:12:33,857 INFO SenderThread:55777 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-30 21:12:34,348 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/wandb-summary.json +2022-01-30 21:12:35,348 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:42,353 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:45,277 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:12:45,278 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:12:48,357 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:12:51,559 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:12:54,360 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:00,364 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:00,477 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:13:00,478 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:13:04,368 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:10,372 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:14,374 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:15,642 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:13:15,643 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:13:18,377 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:21,795 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:13:22,379 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:26,382 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:30,385 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:31,014 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:13:31,014 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:13:37,389 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:41,390 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:45,393 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:46,257 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:13:46,257 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:13:49,395 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:51,396 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:52,019 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:13:55,399 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:13:59,401 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:01,568 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:14:01,569 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:14:03,404 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:07,406 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:09,408 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:13,411 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:15,413 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:16,833 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:14:16,833 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:14:19,415 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:22,297 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:14:23,418 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:25,420 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:29,423 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:32,026 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:14:32,026 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:14:32,425 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:36,428 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:38,430 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:40,431 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:44,435 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:46,437 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:47,191 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:14:47,191 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:14:50,440 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:52,441 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:52,515 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:14:54,443 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:14:56,445 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:00,448 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:02,404 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:15:02,405 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:15:02,450 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:04,452 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:06,454 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:08,456 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:12,459 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:14,461 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:16,463 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:17,574 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:15:17,575 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:15:18,464 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:20,466 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:22,467 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:22,742 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:15:24,469 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:30,473 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:32,778 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:15:32,778 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:15:37,478 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:43,482 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:48,129 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:15:48,129 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:15:49,486 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:52,998 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:15:53,489 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:15:57,492 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:03,444 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:16:03,445 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:16:03,496 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:07,499 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:11,502 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:15,506 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:18,642 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:16:18,642 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:16:20,508 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:23,218 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:16:24,510 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:28,513 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:32,515 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:33,811 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:16:33,811 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:16:36,519 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:40,522 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:44,525 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:48,528 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:48,976 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:16:48,976 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:16:52,530 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:53,521 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:16:54,532 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:16:58,535 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:02,538 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:04,152 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:17:04,152 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:17:06,541 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:08,542 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:13,545 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:15,546 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:19,428 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:17:19,428 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:17:19,549 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:21,550 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:23,766 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:17:25,553 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:27,554 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:31,557 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:33,559 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:34,594 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:17:34,595 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:17:37,561 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:39,562 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:41,564 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:45,566 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:47,568 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:49,569 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:49,754 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:17:49,755 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:17:51,570 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:53,572 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:54,048 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:17:57,575 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:17:59,576 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:01,578 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:03,579 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:04,939 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:18:04,940 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:18:05,581 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:07,583 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:09,584 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:11,586 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:12,541 DEBUG SenderThread:55777 [sender.py:send():234] send: history +2022-01-30 21:18:12,613 DEBUG SenderThread:55777 [sender.py:send():234] send: summary +2022-01-30 21:18:12,643 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/wandb-summary.json +2022-01-30 21:18:12,643 INFO SenderThread:55777 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-30 21:18:13,644 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:15,646 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:20,111 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:18:20,112 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:18:21,651 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:24,375 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:18:27,655 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:34,660 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:35,467 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:18:35,467 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:18:38,663 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:44,667 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:48,670 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:50,634 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:18:50,634 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:18:52,673 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:18:54,618 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:18:58,677 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:02,680 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:05,909 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:19:05,910 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:19:06,683 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:10,686 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:15,689 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:19,692 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:21,160 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:19:21,160 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:19:23,695 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:24,834 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:19:27,697 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:31,700 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:35,703 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:36,362 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:19:36,363 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:19:37,705 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:41,707 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:45,710 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:49,713 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:51,672 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:19:51,672 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:19:51,715 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:55,058 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:19:55,717 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:19:59,720 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:03,723 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:05,724 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:06,988 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:20:06,989 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:20:10,727 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:12,728 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:16,731 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:18,733 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:22,168 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:20:22,168 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:20:22,736 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:24,738 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:25,322 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:20:26,740 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:30,743 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:32,744 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:34,746 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:37,400 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:20:37,400 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:20:38,749 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:40,751 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:42,753 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:44,755 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:46,756 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:48,758 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:52,588 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:20:52,589 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:20:52,761 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:54,763 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:55,616 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:20:56,766 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:20:58,767 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:00,769 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:02,771 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:04,772 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:07,903 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:21:07,904 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:21:13,778 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:19,783 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:23,210 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:21:23,211 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:21:23,786 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:25,918 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:21:29,790 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:33,793 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:38,584 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:21:38,585 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:21:39,797 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:43,799 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:47,802 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:52,806 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:21:53,797 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:21:53,797 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:21:56,220 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:21:58,810 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:02,812 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:06,815 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:09,242 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:22:09,243 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:22:10,818 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:14,819 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:16,821 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:20,823 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:24,616 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:22:24,617 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:22:24,825 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:26,496 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:22:28,828 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:32,831 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:36,834 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:38,836 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:39,890 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:22:39,890 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:22:42,838 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:47,841 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:49,843 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:53,846 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:55,094 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:22:55,095 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:22:55,848 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:22:56,751 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:22:59,851 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:01,852 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:05,855 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:07,857 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:10,414 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:23:10,414 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:23:11,860 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:13,862 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:17,866 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:19,868 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:21,870 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:25,670 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:23:25,670 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:23:25,873 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:26,982 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:23:27,875 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:29,877 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:31,878 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:35,881 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:37,883 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:39,886 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:40,950 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:23:40,951 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:23:41,887 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:43,889 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:45,891 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:47,893 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:49,895 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:51,897 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:53,220 DEBUG SenderThread:55777 [sender.py:send():234] send: history +2022-01-30 21:23:53,329 DEBUG SenderThread:55777 [sender.py:send():234] send: summary +2022-01-30 21:23:53,374 INFO SenderThread:55777 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-30 21:23:53,899 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/wandb-summary.json +2022-01-30 21:23:53,899 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:55,901 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:56,406 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:23:56,406 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:23:56,902 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:23:57,229 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:24:02,906 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:08,910 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:11,631 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:24:11,632 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:24:14,914 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:18,916 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:24,920 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:26,805 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:24:26,805 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:24:27,454 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:24:28,923 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:34,926 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:39,930 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:41,970 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:24:41,971 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:24:43,933 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:47,936 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:51,939 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:55,942 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:24:57,299 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:24:57,299 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:24:57,737 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:24:59,945 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:05,949 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:09,951 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:12,571 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:25:12,572 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:25:13,954 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:15,955 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:19,958 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:23,961 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:27,963 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:27,969 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:25:27,975 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:25:27,975 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:25:32,967 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:34,968 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:38,971 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:40,972 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:43,281 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:25:43,282 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:25:44,974 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:46,976 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:50,978 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:54,981 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:56,982 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:25:58,191 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:25:58,583 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:25:58,584 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:26:00,985 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:02,986 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:04,987 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:08,990 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:10,991 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:12,992 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:13,859 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:26:13,860 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:26:16,995 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:18,996 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:20,999 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:24,001 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:28,003 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:28,486 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:26:29,053 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:26:29,054 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:26:30,005 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:32,006 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:34,007 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:36,009 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:38,010 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:40,011 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:42,012 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:44,014 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:44,205 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:26:44,206 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:26:46,015 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:54,020 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:26:58,693 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:26:59,496 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:26:59,496 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:27:00,024 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:06,028 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:10,030 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:14,935 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:27:14,936 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:27:16,034 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:21,038 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:25,041 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:28,921 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:27:29,044 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:30,253 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:27:30,254 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:27:33,047 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:37,050 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:41,053 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:45,056 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:45,554 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:27:45,555 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:27:49,058 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:53,061 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:57,063 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:27:59,220 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:28:00,854 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:28:00,855 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:28:01,066 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:05,069 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:10,072 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:14,074 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:16,075 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:16,263 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:28:16,264 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:28:20,078 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:24,081 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:26,082 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:29,445 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:28:30,085 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:31,553 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:28:31,554 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:28:34,088 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:36,089 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:40,092 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:42,093 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:46,096 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:46,717 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:28:46,718 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:28:48,098 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:52,101 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:54,102 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:56,103 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:28:59,689 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:29:00,105 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:01,975 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:29:01,976 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:29:02,107 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:04,108 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:06,109 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:10,112 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:12,114 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:15,116 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:17,118 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:17,143 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:29:17,144 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:29:19,119 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:21,121 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:23,122 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:25,123 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:27,124 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:29,126 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:29,989 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:29:31,127 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:32,319 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:29:32,320 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:29:32,731 DEBUG SenderThread:55777 [sender.py:send():234] send: history +2022-01-30 21:29:32,837 DEBUG SenderThread:55777 [sender.py:send():234] send: summary +2022-01-30 21:29:32,890 INFO SenderThread:55777 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-30 21:29:33,128 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/wandb-summary.json +2022-01-30 21:29:33,129 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:34,129 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:35,130 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:37,131 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:41,134 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:45,136 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:47,588 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:29:47,589 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:29:49,139 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:51,141 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:55,144 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:29:58,147 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:00,276 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:30:02,150 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:02,908 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:30:02,908 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:30:06,153 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:08,155 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:12,158 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:14,160 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:18,163 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:18,173 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:30:18,174 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:30:20,165 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:24,168 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:26,170 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:30,173 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:30,488 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:30:33,367 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:30:33,368 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:30:34,176 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:36,178 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:38,181 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:42,183 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:44,185 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:48,188 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:48,574 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:30:48,574 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:30:51,191 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:55,193 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:30:59,196 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:00,819 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:31:01,198 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:03,908 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:31:03,909 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:31:05,202 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:07,204 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:11,207 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:13,209 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:17,212 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:19,293 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:31:19,294 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:31:21,215 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:23,217 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:27,220 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:31,178 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:31:31,223 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:33,224 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:34,579 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:31:34,580 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:31:37,228 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:40,230 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:44,233 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:46,234 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:49,746 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:31:49,747 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:31:50,238 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:52,240 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:56,243 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:31:58,245 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:01,448 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:32:02,248 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:04,250 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:05,058 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:32:05,059 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:32:08,253 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:12,255 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:14,256 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:18,259 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:20,374 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:32:20,375 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:32:22,262 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:24,263 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:26,264 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:30,267 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:31,681 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:32:32,269 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:35,711 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:32:35,712 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:32:37,272 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:39,273 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:43,276 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:47,278 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:49,280 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:51,001 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:32:51,002 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:32:53,283 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:55,284 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:32:59,287 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:01,289 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:01,927 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:33:05,292 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:06,214 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:33:06,214 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:33:07,293 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:11,296 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:15,299 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:17,302 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:21,459 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:33:21,460 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:33:32,174 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:33:36,677 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:33:36,678 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:33:51,905 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:33:51,905 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:33:54,928 DEBUG SenderThread:55777 [sender.py:send():234] send: history +2022-01-30 21:33:54,962 DEBUG SenderThread:55777 [sender.py:send():234] send: summary +2022-01-30 21:33:55,007 INFO SenderThread:55777 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-30 21:33:55,324 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/wandb-summary.json +2022-01-30 21:33:56,324 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:57,325 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:33:59,326 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:34:02,384 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:34:07,099 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:34:07,100 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:34:07,331 INFO Thread-8 :55777 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220130_191954-1mtmnz5y/files/output.log +2022-01-30 21:34:22,261 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:34:22,262 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:34:32,591 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:34:37,427 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:34:37,428 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:34:52,611 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:34:52,611 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status +2022-01-30 21:35:02,828 DEBUG SenderThread:55777 [sender.py:send():234] send: stats +2022-01-30 21:35:07,782 DEBUG HandlerThread:55777 [handler.py:handle_request():130] handle_request: stop_status +2022-01-30 21:35:07,783 DEBUG SenderThread:55777 [sender.py:send_request():248] send_request: stop_status