diff --git "a/checkpoints/wandb/debug-internal.log" "b/checkpoints/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/checkpoints/wandb/debug-internal.log" @@ -0,0 +1,17257 @@ +2024-09-02 17:03:04,766 INFO StreamThr :6573 [internal.py:wandb_internal():85] W&B internal server running at pid: 6573, started at: 2024-09-02 17:03:04.764793 +2024-09-02 17:03:04,769 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status +2024-09-02 17:03:04,778 INFO WriterThread:6573 [datastore.py:open_for_write():87] open: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/run-v43qltex.wandb +2024-09-02 17:03:04,780 DEBUG SenderThread:6573 [sender.py:send():391] send: header +2024-09-02 17:03:04,825 DEBUG SenderThread:6573 [sender.py:send():391] send: run +2024-09-02 17:03:05,497 INFO SenderThread:6573 [dir_watcher.py:__init__():211] watching files in: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files +2024-09-02 17:03:05,498 INFO SenderThread:6573 [sender.py:_start_run_threads():1200] run started: v43qltex with start time 1725296584.761923 +2024-09-02 17:03:05,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: run_start +2024-09-02 17:03:05,573 DEBUG HandlerThread:6573 [system_info.py:__init__():26] System info init +2024-09-02 17:03:05,573 DEBUG HandlerThread:6573 [system_info.py:__init__():41] System info init done +2024-09-02 17:03:05,574 INFO HandlerThread:6573 [system_monitor.py:start():194] Starting system monitor +2024-09-02 17:03:05,574 INFO SystemMonitor:6573 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-02 17:03:05,574 INFO HandlerThread:6573 [system_monitor.py:probe():214] Collecting system info +2024-09-02 17:03:05,576 INFO SystemMonitor:6573 [interfaces.py:start():188] Started cpu monitoring +2024-09-02 17:03:05,577 INFO SystemMonitor:6573 [interfaces.py:start():188] Started disk monitoring +2024-09-02 17:03:05,579 INFO SystemMonitor:6573 [interfaces.py:start():188] Started gpu monitoring +2024-09-02 17:03:05,581 INFO SystemMonitor:6573 [interfaces.py:start():188] Started memory monitoring +2024-09-02 17:03:05,583 INFO SystemMonitor:6573 [interfaces.py:start():188] Started network monitoring +2024-09-02 17:03:05,625 DEBUG HandlerThread:6573 [system_info.py:probe():152] Probing system +2024-09-02 17:03:05,634 DEBUG HandlerThread:6573 [system_info.py:_probe_git():137] Probing git +2024-09-02 17:03:05,666 DEBUG HandlerThread:6573 [system_info.py:_probe_git():145] Probing git done +2024-09-02 17:03:05,667 DEBUG HandlerThread:6573 [system_info.py:probe():200] Probing system done +2024-09-02 17:03:05,667 DEBUG HandlerThread:6573 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-171-generic-x86_64-with-glibc2.35', 'python': '3.11.9', 'heartbeatAt': '2024-09-02T17:03:05.625568', 'startedAt': '2024-09-02T17:03:04.729800', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '-m nanoT5.main', 'codePathLocal': None, 'git': {'remote': 'https://github.com/pszemraj/nanoT5.git', 'commit': '7e55b4db2270303afebba4e0d389b68979943c0c'}, 'email': None, 'root': '/workspace/nanoT5', 'host': 'f8d7d6f6310f', 'username': 'root', 'executable': '/usr/bin/python', 'cpu_count': 48, 'cpu_count_logical': 96, 'cpu_freq': {'current': 1001.9331562499998, 'min': 800.0, 'max': 2801.0}, 'cpu_freq_per_core': [{'current': 900.0, 'min': 800.0, 'max': 2801.0}, {'current': 799.402, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 799.398, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 1037.385, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 872.196, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 1072.759, 'min': 800.0, 'max': 2801.0}, {'current': 882.854, 'min': 800.0, 'max': 2801.0}, {'current': 823.861, 'min': 800.0, 'max': 2801.0}, {'current': 823.212, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 799.39, 'min': 800.0, 'max': 2801.0}, {'current': 823.065, 'min': 800.0, 'max': 2801.0}, {'current': 1398.392, 'min': 800.0, 'max': 2801.0}, {'current': 799.401, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 824.357, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 2022.695, 'min': 800.0, 'max': 2801.0}, {'current': 3300.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 823.663, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 804.641, 'min': 800.0, 'max': 2801.0}, {'current': 873.253, 'min': 800.0, 'max': 2801.0}, {'current': 872.742, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 799.845, 'min': 800.0, 'max': 2801.0}, {'current': 799.422, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 900.0, 'min': 800.0, 'max': 2801.0}, {'current': 799.957, 'min': 800.0, 'max': 2801.0}, {'current': 1100.0, 'min': 800.0, 'max': 2801.0}, {'current': 3351.928, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 799.426, 'min': 800.0, 'max': 2801.0}, {'current': 870.565, 'min': 800.0, 'max': 2801.0}, {'current': 800.192, 'min': 800.0, 'max': 2801.0}, {'current': 800.057, 'min': 800.0, 'max': 2801.0}, {'current': 2799.997, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 959.262, 'min': 800.0, 'max': 2801.0}, {'current': 2801.291, 'min': 800.0, 'max': 2801.0}, {'current': 799.425, 'min': 800.0, 'max': 2801.0}, {'current': 900.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.683, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.697, 'min': 800.0, 'max': 2801.0}, {'current': 800.876, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.741, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 942.364, 'min': 800.0, 'max': 2801.0}, {'current': 800.344, 'min': 800.0, 'max': 2801.0}, {'current': 799.272, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 3300.0, 'min': 800.0, 'max': 2801.0}, {'current': 3304.817, 'min': 800.0, 'max': 2801.0}, {'current': 800.103, 'min': 800.0, 'max': 2801.0}, {'current': 800.363, 'min': 800.0, 'max': 2801.0}, {'current': 800.727, 'min': 800.0, 'max': 2801.0}, {'current': 900.0, 'min': 800.0, 'max': 2801.0}, {'current': 900.0, 'min': 800.0, 'max': 2801.0}, {'current': 900.0, 'min': 800.0, 'max': 2801.0}, {'current': 811.831, 'min': 800.0, 'max': 2801.0}, {'current': 799.938, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 801.226, 'min': 800.0, 'max': 2801.0}, {'current': 799.947, 'min': 800.0, 'max': 2801.0}, {'current': 900.0, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 980.682, 'min': 800.0, 'max': 2801.0}, {'current': 3308.926, 'min': 800.0, 'max': 2801.0}, {'current': 801.074, 'min': 800.0, 'max': 2801.0}, {'current': 800.709, 'min': 800.0, 'max': 2801.0}, {'current': 804.122, 'min': 800.0, 'max': 2801.0}, {'current': 801.051, 'min': 800.0, 'max': 2801.0}, {'current': 805.622, 'min': 800.0, 'max': 2801.0}, {'current': 2800.0, 'min': 800.0, 'max': 2801.0}, {'current': 799.951, 'min': 800.0, 'max': 2801.0}, {'current': 800.0, 'min': 800.0, 'max': 2801.0}, {'current': 2802.488, 'min': 800.0, 'max': 2801.0}, {'current': 801.049, 'min': 800.0, 'max': 2801.0}], 'disk': {'/': {'total': 200.0, 'used': 1.4021186828613281}}, 'gpu': 'NVIDIA A40', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 48305799168}], 'memory': {'total': 503.5313262939453}} +2024-09-02 17:03:05,668 INFO HandlerThread:6573 [system_monitor.py:probe():224] Finished collecting system info +2024-09-02 17:03:05,668 INFO HandlerThread:6573 [system_monitor.py:probe():227] Publishing system info +2024-09-02 17:03:05,672 INFO HandlerThread:6573 [system_monitor.py:probe():229] Finished publishing system info +2024-09-02 17:03:05,676 DEBUG SenderThread:6573 [sender.py:send():391] send: files +2024-09-02 17:03:05,676 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-metadata.json with policy now +2024-09-02 17:03:05,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: python_packages +2024-09-02 17:03:05,821 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:03:05,822 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:03:05,822 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: python_packages +2024-09-02 17:03:05,829 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:03:05,995 DEBUG SenderThread:6573 [sender.py:send():391] send: telemetry +2024-09-02 17:03:06,276 INFO wandb-upload_0:6573 [upload_job.py:push():130] Uploaded file /tmp/tmp9c7fb8h4wandb/1r2a92vs-wandb-metadata.json +2024-09-02 17:03:06,503 INFO Thread-12 :6573 [dir_watcher.py:_on_file_created():271] file/dir created: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/requirements.txt +2024-09-02 17:03:06,506 INFO Thread-12 :6573 [dir_watcher.py:_on_file_created():271] file/dir created: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:06,507 INFO Thread-12 :6573 [dir_watcher.py:_on_file_created():271] file/dir created: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-metadata.json +2024-09-02 17:03:08,519 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:10,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:10,526 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:15,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:15,821 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:03:18,548 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:20,554 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:20,817 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:03:20,818 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:03:21,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:22,560 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:24,565 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:25,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:03:26,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:30,582 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:31,978 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:32,588 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:34,598 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:35,516 DEBUG SenderThread:6573 [sender.py:send():391] send: config +2024-09-02 17:03:35,818 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:03:35,818 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:03:35,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:03:36,613 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:37,018 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:37,616 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/config.yaml +2024-09-02 17:03:37,617 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:03:42,868 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:45,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:03:47,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:50,883 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:03:50,884 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:03:53,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:03:55,883 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:03:58,884 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:03,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:05,584 DEBUG SystemMonitor:6573 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-02 17:04:05,592 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:04:05,884 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:04:05,886 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:04:05,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:04:09,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:14,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:15,884 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:04:20,883 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:04:20,884 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:04:21,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:25,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:04:26,888 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:32,887 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:35,596 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:04:35,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:04:35,887 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:04:35,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:04:38,111 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:43,897 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:45,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:04:48,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:50,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:04:50,886 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:04:54,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:04:55,891 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:04:59,912 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:04,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:05,603 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:05:05,890 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:05:05,892 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:05:05,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:05:10,117 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:15,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:05:15,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:20,892 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:05:20,893 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:05:21,050 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:25,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:05:26,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:31,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:35,608 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:05:35,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:05:35,911 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:05:35,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:05:37,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:42,949 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:45,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:05:48,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:50,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:05:50,908 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:05:54,053 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:05:55,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:05:59,975 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:05,615 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:06:05,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:05,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:06:05,906 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:06:05,967 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:06:10,974 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:15,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:06:15,975 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:20,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:06:20,912 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:06:21,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:25,942 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:06:26,992 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:32,985 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:35,622 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:06:35,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:06:35,908 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:06:35,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:06:38,128 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:43,991 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:45,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:06:48,992 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:50,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:06:50,918 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:06:54,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:06:55,978 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:07:00,006 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:05,628 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:07:05,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:05,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:07:05,922 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:07:05,962 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:07:11,012 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:15,957 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:07:17,009 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:21,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:07:21,390 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:07:22,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:25,951 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:07:28,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:34,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:35,633 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:07:35,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:07:36,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:07:36,345 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:07:39,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:45,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:45,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:07:50,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:51,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:07:51,356 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:07:55,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:07:55,966 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:08:01,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:05,638 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:08:05,976 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:08:06,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:08:06,351 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:08:06,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:12,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:15,982 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:08:17,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:21,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:08:21,351 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:08:22,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:26,003 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:08:28,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:32,628 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:08:34,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:35,646 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:08:35,985 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:08:36,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:08:36,352 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:08:39,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:45,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:45,989 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:08:50,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:51,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:08:51,356 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:08:55,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:08:55,996 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:09:01,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:05,652 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:09:06,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:08,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:09:08,296 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:09:08,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:09:12,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:18,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:18,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:09:23,170 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:23,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:09:23,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:09:28,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:28,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:09:34,171 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:35,655 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:09:38,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:09:38,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:09:38,298 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:09:39,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:45,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:48,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:09:51,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:53,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:09:53,298 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:09:56,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:09:58,171 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:10:02,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:05,662 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:10:07,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:08,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:10:08,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:10:08,298 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:10:13,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:18,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:10:19,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:23,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:10:23,298 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:10:24,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:28,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:10:30,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:35,667 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:10:35,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:38,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:10:38,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:10:38,298 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:10:41,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:47,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:48,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:10:53,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:10:53,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:10:53,298 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:10:58,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:10:59,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:04,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:05,673 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:11:08,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:11:08,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:11:08,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:11:09,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:15,173 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:18,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:11:21,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:23,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:11:23,298 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:11:26,541 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:28,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:11:32,175 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:35,678 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:11:37,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:38,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:11:38,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:11:38,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:11:43,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:48,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:11:49,175 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:53,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:11:53,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:11:54,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:11:58,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:12:00,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:05,681 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:12:05,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:08,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:12:08,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:12:08,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:12:11,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:17,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:18,175 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:12:22,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:23,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:12:23,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:12:27,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:28,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:12:33,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:35,689 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:12:38,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:12:38,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:38,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:12:38,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:12:44,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:48,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:12:50,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:53,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:12:53,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:12:55,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:12:58,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:13:01,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:05,695 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:13:06,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:08,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:13:08,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:13:08,299 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:13:12,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:18,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:18,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:13:23,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:23,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:13:23,300 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:13:28,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:13:29,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:35,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:35,701 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:13:38,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:13:38,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:13:38,300 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:13:40,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:46,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:48,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:13:52,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:53,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:13:53,300 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:13:57,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:13:58,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:14:03,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:05,708 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:14:08,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:14:08,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:08,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:14:08,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:14:14,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:18,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:14:19,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:23,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:14:23,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:14:24,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:28,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:14:30,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:35,711 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:14:35,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:38,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:14:38,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:14:38,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:14:41,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:47,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:48,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:14:53,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:14:53,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:14:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:14:53,691 DEBUG SenderThread:6573 [sender.py:send():391] send: telemetry +2024-09-02 17:14:53,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:14:56,736 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:14:58,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:14:59,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:05,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:05,717 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:15:08,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:15:08,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:15:08,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:15:10,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:16,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:16,813 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/config.yaml +2024-09-02 17:15:18,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:15:22,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:23,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:15:23,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:15:27,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:28,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:15:33,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:35,723 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:15:38,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:15:38,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:38,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:15:38,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:15:44,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:48,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:15:50,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:53,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:15:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:15:55,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:15:58,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:16:01,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:05,728 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:16:06,731 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:08,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:16:08,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:16:08,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:16:12,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:18,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:18,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:16:23,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:23,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:16:23,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:16:28,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:16:29,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:35,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:35,734 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:16:38,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:16:38,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:16:38,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:16:40,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:46,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:48,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:16:51,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:53,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:16:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:16:56,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:16:58,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:17:02,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:05,740 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:17:07,742 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:08,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:17:08,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:17:08,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:17:13,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:18,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:18,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:17:23,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:17:23,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:17:23,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:28,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:17:29,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:34,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:35,746 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:17:38,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:17:38,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:17:38,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:17:39,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:45,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:48,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:17:51,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:53,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:17:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:17:56,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:17:58,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:18:02,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:05,752 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:18:07,756 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:08,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:18:08,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:18:08,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:18:13,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:18,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:18:19,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:23,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:18:23,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:18:24,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:28,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:18:30,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:35,758 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:18:35,763 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:38,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:18:38,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:18:38,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:18:41,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:47,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:48,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:18:52,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:53,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:18:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:18:57,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:18:58,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:19:03,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:05,766 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:19:08,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:19:08,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:08,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:19:08,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:19:14,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:18,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:19:19,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:23,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:19:23,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:19:24,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:28,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:19:30,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:35,770 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:19:35,777 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:38,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:19:38,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:19:38,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:19:41,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:47,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:48,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:19:52,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:53,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:19:53,305 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:19:57,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:19:58,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:20:03,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:05,775 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:20:08,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:08,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:20:08,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:20:08,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:20:14,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:18,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:20:19,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:23,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:20:23,305 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:20:23,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:20:23,710 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:20:23,710 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:20:23,713 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:20:24,144 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:20:24,145 INFO Thread-12 :6573 [dir_watcher.py:_on_file_created():271] file/dir created: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:20:24,715 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:28,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:20:30,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:35,781 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:20:35,786 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:38,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:20:38,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:20:38,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:20:41,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:47,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:48,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:20:52,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:53,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:20:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:20:57,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:20:58,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:21:03,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:05,787 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:21:08,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:21:08,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:21:08,305 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:21:08,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:14,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:18,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:21:20,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:23,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:21:23,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:21:25,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:28,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:21:31,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:35,792 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:21:36,795 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:38,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:21:38,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:21:38,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:21:42,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:48,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:48,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:21:53,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:21:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:21:53,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:21:58,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:21:59,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:05,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:05,795 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:22:08,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:22:08,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:22:08,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:22:10,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:16,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:18,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:22:21,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:23,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:22:23,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:22:26,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:28,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:22:32,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:35,799 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:22:37,802 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:38,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:22:38,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:22:38,305 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:22:43,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:48,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:22:48,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:53,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:22:53,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:22:53,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:22:58,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:22:59,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:05,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:05,805 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:23:08,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:23:08,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:23:08,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:23:10,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:16,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:18,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:23:22,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:23,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:23:23,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:23:27,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:28,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:23:33,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:35,809 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:23:38,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:23:38,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:23:38,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:23:38,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:44,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:48,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:23:49,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:53,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:23:53,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:23:54,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:23:58,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:24:00,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:05,817 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:24:05,822 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:08,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:24:08,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:24:08,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:24:11,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:17,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:18,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:24:22,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:23,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:24:23,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:24:27,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:28,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:24:33,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:35,821 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:24:38,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:24:38,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:38,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:24:38,306 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:24:44,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:48,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:24:49,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:53,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:24:53,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:24:54,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:24:58,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:25:00,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:05,827 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:25:05,833 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:08,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:25:08,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:25:08,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:25:11,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:17,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:18,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:25:22,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:23,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:25:23,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:25:27,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:28,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:25:33,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:35,833 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:25:38,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:25:38,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:25:38,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:25:38,542 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:44,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:48,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:25:50,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:53,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:25:53,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:25:54,840 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:25:54,844 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:25:54,844 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:25:54,848 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:25:55,500 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:25:55,850 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:25:56,508 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:25:58,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:26:01,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:05,839 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:26:06,842 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:08,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:26:08,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:26:08,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:26:12,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:18,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:18,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:26:23,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:26:23,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:26:23,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:28,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:26:29,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:35,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:35,844 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:26:38,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:26:38,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:26:38,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:26:40,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:46,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:48,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:26:51,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:53,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:26:53,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:26:56,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:26:58,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:27:02,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:05,848 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:27:07,850 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:08,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:27:08,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:27:08,309 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:27:13,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:18,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:18,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:27:23,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:27:23,307 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:27:23,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:28,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:27:29,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:35,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:35,854 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:27:38,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:27:38,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:27:38,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:27:40,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:46,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:48,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:27:51,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:53,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:27:53,309 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:27:56,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:27:58,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:28:02,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:05,860 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:28:07,863 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:08,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:28:08,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:28:08,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:28:13,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:18,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:28:18,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:23,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:28:23,308 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:28:23,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:28,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:28:29,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:35,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:35,867 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:28:38,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:28:38,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:28:38,315 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:28:40,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:46,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:48,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:28:51,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:53,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:28:53,309 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:28:56,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:28:58,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:29:02,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:05,871 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:29:07,874 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:08,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:29:08,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:29:08,309 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:29:13,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:18,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:29:18,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:23,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:29:23,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:29:23,541 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:28,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:29:29,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:35,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:35,877 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:29:38,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:29:38,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:29:38,309 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:29:40,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:46,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:48,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:29:52,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:53,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:29:53,309 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:29:57,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:29:58,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:30:03,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:05,883 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:30:08,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:30:08,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:08,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:30:08,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:30:14,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:18,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:30:19,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:23,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:30:23,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:30:24,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:28,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:30:30,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:35,888 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:30:35,894 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:38,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:30:38,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:30:38,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:30:41,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:47,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:48,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:30:53,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:30:53,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:30:53,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:30:58,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:30:59,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:05,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:05,894 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:31:08,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:31:08,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:31:08,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:31:10,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:16,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:18,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:31:22,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:23,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:31:23,310 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:31:26,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:31:26,100 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:31:26,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:31:26,116 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:31:26,833 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:31:26,834 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:31:28,118 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:28,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:31:33,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:35,901 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:31:38,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:31:38,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:38,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:31:38,311 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:31:44,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:48,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:31:49,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:53,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:31:53,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:31:54,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:31:58,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:32:00,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:05,906 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:32:05,912 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:08,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:32:08,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:32:08,311 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:32:11,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:17,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:18,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:32:22,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:23,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:32:23,311 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:32:27,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:28,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:32:33,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:35,912 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:32:38,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:32:38,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:38,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:32:38,311 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:32:44,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:48,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:32:50,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:53,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:32:53,311 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:32:55,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:32:58,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:33:01,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:05,918 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:33:06,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:08,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:33:08,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:33:08,311 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:33:12,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:18,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:18,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:33:23,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:23,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:33:23,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:33:28,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:33:29,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:35,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:35,924 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:33:38,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:33:38,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:33:38,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:33:40,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:46,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:48,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:33:51,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:53,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:33:53,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:33:56,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:33:58,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:34:02,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:05,965 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:34:07,968 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:08,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:34:08,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:34:08,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:34:13,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:18,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:18,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:34:23,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:34:23,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:34:23,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:28,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:34:29,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:35,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:35,971 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:34:38,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:34:38,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:34:38,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:34:40,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:46,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:48,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:34:51,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:53,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:34:53,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:34:56,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:34:58,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:35:02,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:05,977 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:35:07,981 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:08,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:35:08,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:35:08,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:35:13,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:18,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:35:18,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:23,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:35:23,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:35:23,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:28,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:35:29,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:35,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:35,983 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:35:38,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:35:38,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:35:38,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:35:40,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:46,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:48,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:35:51,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:53,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:35:53,314 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:35:56,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:35:58,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:36:02,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:05,989 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:36:07,992 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:08,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:36:08,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:36:08,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:36:13,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:18,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:36:18,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:23,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:36:23,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:36:23,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:28,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:36:29,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:35,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:35,995 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:36:38,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:36:38,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:36:38,314 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:36:40,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:46,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:48,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:36:52,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:53,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:36:53,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:36:56,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:36:56,195 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:36:56,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:36:56,199 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:36:57,184 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:36:57,186 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:36:58,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:36:58,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:37:03,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:06,001 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:37:08,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:37:08,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:37:08,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:37:08,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:14,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:18,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:37:20,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:23,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:37:23,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:37:25,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:28,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:37:31,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:36,004 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:37:37,006 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:38,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:37:38,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:37:38,314 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:37:42,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:48,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:48,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:37:53,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:37:53,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:37:53,314 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:37:58,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:37:59,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:05,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:06,015 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:38:08,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:38:08,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:38:08,314 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:38:10,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:16,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:18,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:38:22,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:23,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:38:23,315 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:38:27,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:28,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:38:33,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:36,019 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:38:38,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:38:38,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:38:38,314 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:38:38,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:44,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:48,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:38:50,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:53,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:38:53,315 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:38:55,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:38:58,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:39:01,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:06,024 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:39:07,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:08,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:39:08,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:39:08,315 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:39:12,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:18,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:18,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:39:23,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:39:23,315 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:39:23,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:28,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:39:29,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:35,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:36,029 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:39:39,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:39:39,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:39:39,359 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:39:40,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:46,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:49,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:39:51,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:54,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:39:54,357 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:39:56,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:39:59,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:40:02,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:06,035 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:40:08,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:09,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:40:09,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:40:09,358 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:40:13,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:19,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:40:19,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:24,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:40:24,356 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:40:24,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:29,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:40:30,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:35,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:36,041 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:40:39,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:40:39,357 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:40:39,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:40:40,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:46,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:49,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:40:51,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:54,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:40:54,357 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:40:56,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:40:59,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:41:02,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:06,044 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:41:08,047 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:09,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:41:09,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:41:09,359 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:41:13,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:19,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:41:19,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:24,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:41:24,357 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:41:24,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:29,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:41:30,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:36,049 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:41:36,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:39,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:41:39,358 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:41:39,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:41:41,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:47,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:49,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:41:52,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:54,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:41:54,358 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:41:57,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:41:59,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:42:03,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:06,054 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:42:09,057 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:09,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:42:09,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:42:09,359 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:42:14,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:19,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:42:19,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:24,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:42:24,358 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:42:24,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:27,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:42:27,698 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:42:27,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:42:27,712 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:42:28,677 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:42:29,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:42:29,681 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:42:30,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:35,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:36,059 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:42:39,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:42:39,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:42:39,361 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:42:40,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:46,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:49,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:42:52,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:54,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:42:54,359 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:42:57,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:42:59,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:43:03,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:06,067 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:43:09,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:09,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:43:09,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:43:09,361 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:43:14,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:19,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:43:20,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:24,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:43:24,359 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:43:25,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:29,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:43:31,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:36,071 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:43:37,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:39,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:43:39,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:43:39,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:43:42,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:48,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:49,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:43:54,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:43:54,365 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:43:54,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:43:59,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:44:00,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:05,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:06,076 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:44:09,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:44:09,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:44:09,361 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:44:10,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:16,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:19,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:44:22,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:24,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:44:24,362 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:44:27,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:29,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:44:33,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:36,082 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:44:39,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:39,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:44:39,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:44:39,361 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:44:44,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:49,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:44:50,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:54,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:44:54,361 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:44:55,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:44:59,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:45:01,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:06,087 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:45:07,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:09,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:45:09,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:45:09,363 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:45:12,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:18,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:19,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:45:24,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:45:24,362 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:45:24,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:29,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:45:30,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:35,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:36,093 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:45:39,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:45:39,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:45:39,363 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:45:40,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:46,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:49,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:45:52,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:54,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:45:54,362 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:45:57,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:45:59,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:46:03,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:06,097 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:46:09,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:09,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:46:09,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:46:09,363 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:46:14,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:19,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:46:20,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:24,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:46:24,362 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:46:25,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:29,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:46:31,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:36,103 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:46:37,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:39,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:46:39,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:46:39,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:46:42,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:48,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:49,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:46:54,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:46:54,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:46:54,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:46:59,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:47:00,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:05,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:06,107 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:47:09,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:47:09,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:47:09,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:47:10,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:16,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:19,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:47:22,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:24,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:47:24,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:47:27,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:29,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:47:33,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:36,113 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:47:39,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:39,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:47:39,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:47:39,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:47:44,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:49,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:47:50,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:54,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:47:54,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:47:55,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:47:57,967 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:47:57,971 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:47:57,972 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:47:57,977 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:47:57,981 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:47:59,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:47:59,987 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:48:01,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:06,121 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:48:07,125 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:09,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:48:09,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:48:09,365 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:48:12,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:18,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:19,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:48:24,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:48:24,365 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:48:24,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:29,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:48:30,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:36,123 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:48:36,128 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:39,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:48:39,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:48:39,366 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:48:42,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:47,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:49,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:48:53,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:54,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:48:54,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:48:58,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:48:59,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:49:04,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:06,129 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:49:09,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:49:09,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:49:09,365 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:49:09,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:15,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:19,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:49:21,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:24,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:49:24,364 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:49:26,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:29,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:49:32,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:36,134 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:49:38,137 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:39,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:49:39,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:49:39,366 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:49:43,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:49,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:49:49,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:54,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:49:54,365 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:49:54,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:49:59,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:50:00,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:06,139 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:50:06,143 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:09,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:50:09,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:50:09,366 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:50:11,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:17,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:19,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:50:23,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:24,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:50:24,366 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:50:28,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:29,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:50:34,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:36,145 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:50:39,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:50:39,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:50:39,367 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:50:39,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:45,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:49,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:50:51,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:54,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:50:54,366 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:50:56,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:50:59,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:51:02,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:06,150 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:51:08,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:09,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:51:09,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:51:09,367 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:51:13,578 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:19,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:51:19,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:24,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:51:24,367 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:51:24,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:29,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:51:30,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:35,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:36,156 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:51:39,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:51:39,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:51:39,370 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:51:40,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:46,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:49,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:51:52,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:54,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:51:54,367 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:51:57,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:51:59,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:52:03,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:06,159 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:52:09,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:09,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:52:09,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:52:09,369 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:52:14,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:19,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:52:20,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:24,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:52:24,367 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:52:25,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:29,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:52:31,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:36,165 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:52:37,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:39,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:52:39,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:52:39,368 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:52:42,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:48,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:49,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:52:54,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:52:54,368 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:52:54,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:52:59,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:53:00,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:05,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:06,172 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:53:09,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:53:09,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:53:09,368 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:53:10,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:16,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:19,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:53:22,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:24,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:53:24,368 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:53:27,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:29,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:53:29,233 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:53:29,233 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:53:29,237 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:53:29,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:53:29,450 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:53:31,455 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:53:33,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:36,178 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:53:39,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:39,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:53:39,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:53:39,370 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:53:44,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:49,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:53:50,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:54,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:53:54,369 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:53:55,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:53:59,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:54:01,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:06,183 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:54:07,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:09,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:54:09,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:54:09,370 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:54:12,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:18,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:19,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:54:24,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:54:24,371 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:54:24,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:29,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:54:30,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:35,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:36,189 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:54:39,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:54:39,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:54:39,370 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:54:40,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:46,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:49,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:54:52,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:54,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:54:54,371 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:54:57,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:54:59,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:55:03,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:06,195 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:55:09,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:09,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:55:09,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:55:09,371 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:55:14,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:19,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:55:20,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:24,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:55:24,371 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:55:25,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:29,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:55:31,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:36,200 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:55:37,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:39,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:55:39,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:55:39,371 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:55:42,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:48,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:49,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:55:54,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:55:54,371 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:55:54,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:55:59,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:56:00,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:06,203 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:56:06,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:09,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:56:09,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:56:09,372 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:56:11,578 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:17,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:19,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:56:23,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:24,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:56:24,372 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:56:28,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:29,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:56:34,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:36,211 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:56:39,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:56:39,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:56:39,375 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:56:39,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:45,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:49,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:56:51,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:54,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:56:54,371 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:56:56,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:56:59,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:57:02,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:06,215 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:57:08,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:09,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:57:09,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:57:09,374 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:57:13,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:19,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:57:19,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:24,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:57:24,372 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:57:24,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:29,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:57:30,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:36,221 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:57:36,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:39,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:57:39,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:57:39,376 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:57:41,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:47,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:49,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:57:53,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:54,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:57:54,374 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:57:58,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:57:59,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:58:04,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:06,230 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:58:09,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:58:09,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:58:09,375 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:58:09,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:15,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:19,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:58:21,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:24,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:58:24,374 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:58:26,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:29,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:58:32,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:36,229 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:58:38,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:39,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:58:39,374 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:58:39,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:58:43,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:49,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:58:49,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:54,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:58:54,374 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:58:54,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:58:59,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:59:00,012 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 17:59:00,015 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 17:59:00,016 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 17:59:00,019 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:00,024 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 17:59:00,612 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 17:59:01,623 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 17:59:05,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:06,235 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:59:09,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:59:09,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:59:09,377 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:59:10,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:16,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:19,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:59:22,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:24,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:59:24,376 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:59:27,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:29,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:59:33,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:36,244 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 17:59:39,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:39,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:59:39,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:59:39,377 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:59:44,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:49,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 17:59:50,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:54,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 17:59:54,377 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 17:59:55,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 17:59:59,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:00:01,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:06,246 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:00:07,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:09,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:00:09,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:00:09,378 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:00:12,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:18,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:19,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:00:24,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:00:24,377 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:00:24,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:29,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:00:30,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:35,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:36,250 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:00:39,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:00:39,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:00:39,379 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:00:40,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:46,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:49,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:00:52,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:54,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:00:54,378 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:00:57,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:00:59,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:01:03,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:06,256 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:01:09,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:09,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:01:09,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:01:09,380 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:01:14,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:19,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:01:20,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:24,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:01:24,378 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:01:25,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:29,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:01:31,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:36,260 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:01:37,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:39,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:01:39,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:01:39,379 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:01:42,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:48,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:49,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:01:53,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:54,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:01:54,379 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:01:58,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:01:59,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:02:04,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:06,263 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:02:09,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:02:09,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:02:09,379 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:02:09,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:15,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:19,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:02:21,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:24,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:02:24,379 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:02:26,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:29,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:02:32,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:36,270 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:02:38,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:39,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:02:39,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:02:39,379 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:02:43,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:49,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:02:49,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:54,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:02:54,379 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:02:54,572 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:02:59,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:03:00,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:05,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:06,277 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:03:09,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:03:09,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:03:09,382 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:03:10,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:16,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:19,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:03:22,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:24,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:03:24,380 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:03:27,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:29,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:03:33,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:36,280 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:03:39,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:39,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:03:39,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:03:39,381 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:03:44,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:49,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:03:50,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:54,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:03:54,381 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:03:55,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:03:59,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:04:01,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:06,286 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:04:07,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:09,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:04:09,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:04:09,381 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:04:12,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:18,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:19,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:04:24,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:04:24,382 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:04:24,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:29,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:04:30,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:04:30,280 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:04:30,282 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:04:30,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:30,291 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:04:30,826 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:04:31,838 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:04:35,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:36,292 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:04:39,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:04:39,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:04:39,382 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:04:40,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:46,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:49,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:04:52,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:54,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:04:54,382 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:04:57,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:04:59,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:05:03,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:06,296 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:05:09,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:09,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:05:09,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:05:09,383 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:05:14,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:19,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:05:20,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:24,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:05:24,381 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:05:25,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:29,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:05:31,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:36,301 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:05:37,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:39,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:05:39,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:05:39,382 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:05:42,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:48,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:49,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:05:54,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:05:54,383 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:05:54,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:05:59,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:06:00,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:06,306 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:06:06,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:09,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:06:09,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:06:09,383 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:06:11,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:17,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:19,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:06:22,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:24,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:06:24,382 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:06:27,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:29,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:06:33,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:36,313 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:06:39,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:39,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:06:39,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:06:39,384 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:06:44,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:49,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:06:50,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:54,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:06:54,383 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:06:55,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:06:59,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:07:01,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:06,319 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:07:07,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:09,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:07:09,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:07:09,386 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:07:12,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:18,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:19,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:07:24,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:07:24,383 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:07:24,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:29,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:07:30,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:35,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:36,328 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:07:39,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:07:39,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:07:39,385 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:07:40,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:46,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:49,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:07:52,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:54,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:07:54,383 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:07:57,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:07:59,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:08:03,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:06,331 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:08:09,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:09,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:08:09,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:08:09,385 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:08:14,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:19,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:08:20,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:24,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:08:24,385 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:08:25,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:29,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:08:31,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:36,337 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:08:37,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:39,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:08:39,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:08:39,386 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:08:42,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:48,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:49,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:08:54,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:08:54,384 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:08:54,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:08:59,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:09:00,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:05,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:06,343 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:09:09,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:09:09,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:09:09,387 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:09:10,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:16,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:19,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:09:22,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:24,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:09:24,385 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:09:27,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:29,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:09:33,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:36,348 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:09:39,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:39,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:09:39,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:09:39,387 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:09:44,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:49,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:09:50,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:54,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:09:54,385 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:09:55,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:09:59,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:10:01,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:01,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:10:01,474 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:10:01,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:10:01,484 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:10:01,485 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:10:04,529 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:10:06,354 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:10:07,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:09,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:10:09,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:10:09,388 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:10:12,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:18,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:19,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:10:24,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:10:24,386 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:10:24,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:29,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:10:30,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:35,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:36,359 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:10:39,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:10:39,390 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:10:39,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:10:40,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:46,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:49,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:10:52,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:54,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:10:54,389 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:10:57,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:10:59,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:11:03,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:06,365 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:11:09,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:09,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:11:09,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:11:09,389 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:11:14,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:19,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:11:20,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:24,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:11:24,390 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:11:25,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:29,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:11:31,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:36,372 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:11:37,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:39,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:11:39,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:11:39,390 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:11:42,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:48,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:49,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:11:54,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:11:54,389 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:11:54,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:11:59,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:12:00,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:05,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:06,377 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:12:09,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:12:09,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:12:09,390 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:12:10,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:16,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:19,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:12:21,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:24,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:12:24,390 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:12:26,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:29,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:12:32,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:36,385 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:12:38,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:39,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:12:39,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:12:39,389 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:12:43,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:49,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:12:49,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:54,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:12:54,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:12:54,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:12:59,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:13:00,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:05,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:06,390 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:13:09,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:13:09,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:13:09,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:13:10,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:16,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:19,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:13:22,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:24,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:13:24,390 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:13:27,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:29,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:13:33,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:36,396 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:13:39,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:13:39,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:13:39,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:13:39,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:45,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:49,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:13:51,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:54,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:13:54,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:13:56,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:13:59,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:14:02,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:06,402 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:14:08,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:09,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:14:09,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:14:09,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:14:13,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:19,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:14:19,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:24,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:14:24,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:14:24,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:29,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:14:30,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:36,407 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:14:36,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:39,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:14:39,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:14:39,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:14:41,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:47,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:49,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:14:53,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:54,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:14:54,392 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:14:58,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:14:59,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:15:04,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:06,412 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:15:09,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:15:09,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:15:09,393 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:15:09,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:15,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:19,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:15:21,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:24,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:15:24,392 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:15:26,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:29,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:15:32,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:33,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:15:33,368 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:15:33,369 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:15:33,377 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:15:33,852 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:15:34,855 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:15:36,420 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:15:38,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:39,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:15:39,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:15:39,391 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:15:43,541 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:49,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:15:49,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:54,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:15:54,392 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:15:54,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:15:59,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:16:00,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:05,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:06,424 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:16:09,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:16:09,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:16:09,394 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:16:10,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:16,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:19,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:16:22,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:24,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:16:24,393 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:16:27,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:29,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:16:33,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:36,430 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:16:38,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:39,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:16:39,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:16:39,392 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:16:43,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:49,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:16:49,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:54,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:16:54,397 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:16:54,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:16:59,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:17:00,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:06,432 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:17:06,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:09,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:17:09,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:17:09,394 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:17:11,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:17,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:19,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:17:23,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:24,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:17:24,394 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:17:28,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:29,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:17:34,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:36,443 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:17:39,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:17:39,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:17:39,394 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:17:39,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:45,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:49,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:17:50,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:54,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:17:54,394 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:17:55,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:17:59,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:18:01,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:06,446 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:18:06,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:09,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:18:09,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:18:09,395 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:18:11,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:17,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:19,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:18:23,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:24,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:18:24,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:18:28,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:29,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:18:34,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:36,451 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:18:39,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:18:39,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:18:39,395 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:18:39,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:45,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:49,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:18:51,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:54,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:18:54,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:18:56,576 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:18:59,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:19:02,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:06,456 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:19:07,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:09,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:19:09,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:19:09,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:19:12,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:18,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:19,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:19:24,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:19:24,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:19:24,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:29,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:19:30,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:35,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:36,463 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:19:39,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:19:39,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:19:39,395 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:19:40,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:46,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:49,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:19:52,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:54,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:19:54,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:19:57,573 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:19:59,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:20:03,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:06,467 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:20:08,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:09,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:20:09,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:20:09,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:20:13,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:19,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:20:19,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:24,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:20:24,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:20:24,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:29,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:20:30,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:35,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:36,474 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:20:39,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:20:39,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:20:39,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:20:40,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:46,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:49,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:20:52,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:54,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:20:54,397 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:20:57,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:20:59,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:21:03,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:03,903 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:21:03,909 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:21:03,909 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:21:03,913 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:21:04,217 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:21:06,479 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:21:07,226 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:21:08,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:09,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:21:09,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:21:09,397 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:21:13,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:19,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:21:19,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:24,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:21:24,397 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:21:24,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:29,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:21:30,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:35,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:36,481 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:21:39,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:21:39,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:21:39,396 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:21:40,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:46,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:49,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:21:52,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:54,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:21:54,397 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:21:57,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:21:59,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:22:03,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:06,488 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:22:08,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:09,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:22:09,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:22:09,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:22:13,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:19,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:22:19,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:24,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:22:24,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:22:24,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:29,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:22:30,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:36,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:36,494 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:22:39,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:22:39,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:22:39,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:22:41,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:47,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:49,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:22:53,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:54,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:22:54,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:22:58,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:22:59,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:23:04,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:06,498 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:23:09,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:23:09,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:23:09,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:23:09,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:15,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:19,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:23:20,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:24,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:23:24,399 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:23:25,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:29,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:23:31,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:36,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:36,504 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:23:39,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:23:39,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:23:39,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:23:41,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:47,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:49,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:23:53,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:54,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:23:54,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:23:58,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:23:59,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:24:04,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:06,509 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:24:09,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:24:09,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:24:09,412 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:24:09,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:15,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:19,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:24:21,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:24,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:24:24,399 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:24:26,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:29,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:24:32,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:36,515 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:24:37,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:39,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:24:39,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:24:39,408 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:24:42,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:48,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:49,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:24:54,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:24:54,400 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:24:54,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:24:59,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:25:00,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:06,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:06,521 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:25:09,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:25:09,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:25:09,406 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:25:11,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:17,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:19,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:25:23,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:24,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:25:24,400 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:25:28,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:29,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:25:34,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:36,526 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:25:39,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:25:39,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:25:39,399 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:25:39,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:45,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:49,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:25:50,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:54,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:25:54,406 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:25:55,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:25:59,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:26:01,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:06,532 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:26:06,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:09,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:26:09,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:26:09,416 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:26:11,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:17,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:19,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:26:22,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:24,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:26:24,400 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:26:27,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:29,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:26:33,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:35,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:26:35,353 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:26:35,354 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:26:35,357 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:26:36,016 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:26:36,537 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:26:37,019 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:26:38,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:39,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:26:39,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:26:39,407 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:26:43,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:49,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:26:49,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:54,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:26:54,411 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:26:54,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:26:59,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:27:00,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:06,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:06,542 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:27:09,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:27:09,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:27:09,407 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:27:11,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:17,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:19,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:27:22,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:24,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:27:24,406 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:27:27,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:29,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:27:33,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:36,549 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:27:38,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:39,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:27:39,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:27:39,412 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:27:43,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:49,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:27:49,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:54,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:27:54,407 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:27:54,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:27:59,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:28:00,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:06,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:06,554 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:28:09,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:28:09,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:28:09,413 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:28:11,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:17,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:19,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:28:23,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:24,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:28:24,406 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:28:28,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:29,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:28:34,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:36,559 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:28:39,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:28:39,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:28:39,411 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:28:39,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:45,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:49,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:28:51,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:54,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:28:54,409 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:28:56,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:28:59,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:29:02,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:06,565 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:29:07,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:09,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:29:09,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:29:09,409 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:29:12,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:18,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:19,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:29:24,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:29:24,406 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:29:24,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:29,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:29:30,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:35,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:36,571 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:29:39,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:29:39,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:29:39,408 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:29:40,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:46,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:49,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:29:52,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:54,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:29:54,407 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:29:57,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:29:59,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:30:03,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:06,577 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:30:08,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:09,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:30:09,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:30:09,413 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:30:13,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:19,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:30:19,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:24,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:30:24,411 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:30:24,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:29,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:30:30,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:35,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:36,583 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:30:39,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:30:39,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:30:39,410 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:30:40,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:46,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:49,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:30:52,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:54,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:30:54,411 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:30:57,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:30:59,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:31:03,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:06,588 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:31:08,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:09,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:31:09,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:31:09,411 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:31:14,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:19,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:31:20,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:24,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:31:24,410 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:31:25,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:29,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:31:31,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:36,594 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:31:36,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:39,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:31:39,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:31:39,410 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:31:41,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:47,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:49,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:31:53,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:54,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:31:54,410 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:31:58,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:31:59,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:32:04,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:06,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:32:06,053 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:32:06,054 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:32:06,060 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:32:06,608 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:32:06,678 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:32:06,680 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:32:09,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:32:09,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:32:09,412 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:32:09,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:15,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:19,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:32:21,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:25,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:32:25,573 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:32:26,754 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:29,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:32:32,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:36,605 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:32:37,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:39,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:32:40,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:32:40,568 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:32:42,814 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:48,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:49,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:32:54,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:32:55,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:32:55,570 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:32:59,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:32:59,817 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:05,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:06,614 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:33:09,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:33:10,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:10,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:33:10,569 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:33:15,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:19,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:33:21,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:25,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:33:25,571 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:33:26,791 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:29,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:33:32,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:36,617 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:33:37,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:39,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:33:40,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:33:40,571 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:33:42,774 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:48,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:49,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:33:54,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:33:55,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:33:55,571 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:33:59,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:33:59,780 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:05,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:06,623 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:34:09,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:34:10,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:34:10,570 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:34:10,731 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:16,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:19,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:34:22,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:25,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:34:25,572 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:34:27,791 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:29,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:34:33,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:36,629 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:34:38,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:39,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:34:40,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:34:40,572 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:34:43,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:49,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:34:49,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:54,572 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:34:55,572 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:34:55,574 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:34:59,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:34:59,732 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:05,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:06,633 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:35:09,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:35:10,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:10,573 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:35:10,573 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:35:15,572 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:19,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:35:21,572 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:25,575 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:35:25,576 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:35:26,800 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:29,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:35:32,574 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:36,640 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:35:37,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:39,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:35:40,574 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:35:40,575 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:35:42,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:48,575 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:49,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:35:54,575 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:35:55,575 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:35:55,577 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:35:59,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:35:59,731 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:05,575 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:06,646 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:36:09,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:36:10,576 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:10,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:36:10,578 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:36:16,575 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:19,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:36:21,576 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:25,576 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:36:25,578 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:36:26,732 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:29,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:36:32,576 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:36,651 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:36:37,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:39,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:36:40,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:36:40,579 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:36:42,819 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:48,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:49,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:36:54,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:36:55,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:36:55,579 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:36:59,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:36:59,811 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:05,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:06,664 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:37:09,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:37:10,578 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:37:10,580 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:37:10,788 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:16,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:19,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:37:22,577 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:25,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:37:25,580 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:37:27,811 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:29,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:37:33,578 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:36,663 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:37:37,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:37:37,909 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:37:37,909 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:37:37,916 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:37:38,225 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:37:38,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:39,228 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:37:39,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:37:40,578 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:37:40,579 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:37:44,824 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:49,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:37:50,578 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:55,578 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:37:55,579 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:37:55,742 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:37:59,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:38:01,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:06,669 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:38:06,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:09,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:38:10,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:38:10,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:38:11,802 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:17,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:19,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:38:23,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:25,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:38:25,580 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:38:28,764 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:29,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:38:34,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:36,677 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:38:39,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:38:39,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:40,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:38:40,580 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:38:44,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:49,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:38:50,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:55,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:38:55,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:38:55,728 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:38:59,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:39:01,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:06,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:06,680 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:39:09,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:39:10,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:39:10,579 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:39:11,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:17,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:19,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:39:23,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:25,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:39:25,580 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:39:28,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:29,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:39:34,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:36,685 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:39:39,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:39:39,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:40,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:39:40,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:39:44,787 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:49,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:39:50,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:55,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:39:55,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:39:55,735 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:39:59,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:40:01,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:06,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:06,691 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:40:09,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:40:10,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:40:10,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:40:11,760 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:17,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:19,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:40:23,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:25,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:40:25,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:40:28,747 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:29,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:40:34,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:36,696 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:40:39,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:40:39,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:40,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:40:40,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:40:44,781 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:49,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:40:50,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:55,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:40:55,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:40:55,798 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:40:59,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:41:01,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:06,696 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:41:06,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:09,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:41:10,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:41:10,582 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:41:11,821 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:17,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:19,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:41:23,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:25,580 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:41:25,581 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:41:28,760 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:29,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:41:34,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:36,702 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:41:39,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:41:39,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:40,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:41:40,582 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:41:44,757 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:49,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:41:50,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:55,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:41:55,584 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:41:55,791 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:41:59,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:42:01,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:06,710 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:42:06,713 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:09,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:42:10,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:42:10,583 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:42:11,822 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:17,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:19,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:42:22,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:25,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:42:25,585 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:42:27,810 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:29,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:42:33,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:36,715 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:42:38,718 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:39,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:42:40,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:42:40,583 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:42:43,831 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:49,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:42:49,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:55,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:42:55,585 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:42:55,755 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:42:59,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:43:01,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:06,720 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:43:06,723 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:09,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:43:09,723 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:43:09,727 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:43:09,729 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:43:09,735 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:43:10,144 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:43:10,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:43:10,583 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:43:11,159 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:43:11,806 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:17,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:19,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:43:23,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:25,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:43:25,584 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:43:28,745 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:29,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:43:34,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:36,726 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:43:39,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:43:39,729 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:40,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:43:40,585 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:43:44,767 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:49,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:43:50,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:55,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:43:55,585 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:43:55,773 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:43:59,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:44:01,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:06,731 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:44:06,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:09,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:44:10,583 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:44:10,585 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:44:11,825 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:17,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:19,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:44:23,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:25,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:44:25,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:44:28,750 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:29,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:44:34,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:36,738 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:44:39,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:44:39,741 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:40,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:44:40,585 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:44:44,792 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:49,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:44:50,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:55,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:44:55,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:44:55,755 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:44:59,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:45:01,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:06,743 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:45:06,748 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:09,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:45:10,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:45:10,585 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:45:11,813 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:17,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:19,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:45:22,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:25,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:45:25,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:45:27,804 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:29,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:45:33,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:36,746 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:45:38,748 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:39,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:45:40,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:45:40,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:45:43,797 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:49,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:45:49,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:55,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:45:55,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:45:55,761 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:45:59,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:46:01,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:06,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:06,750 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:46:09,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:46:10,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:46:10,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:46:11,799 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:17,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:19,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:46:23,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:25,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:46:25,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:46:28,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:29,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:46:34,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:36,761 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:46:39,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:46:39,764 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:40,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:46:40,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:46:44,824 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:49,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:46:50,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:55,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:46:55,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:46:55,823 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:46:59,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:47:01,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:06,761 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:47:06,763 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:09,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:47:10,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:47:10,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:47:11,804 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:17,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:19,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:47:23,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:25,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:47:25,587 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:47:28,799 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:29,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:47:34,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:36,767 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:47:39,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:47:39,770 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:40,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:47:40,586 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:47:44,784 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:49,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:47:50,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:55,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:47:55,587 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:47:55,811 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:47:59,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:48:01,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:06,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:06,769 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:48:09,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:48:10,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:48:10,587 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:48:11,794 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:17,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:19,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:48:23,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:25,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:48:25,587 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:48:28,753 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:29,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:48:34,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:36,776 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:48:39,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:48:39,780 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:40,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:48:40,597 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:48:40,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:48:40,914 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:48:40,915 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:48:40,921 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:48:41,044 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:48:44,065 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:48:44,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:49,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:48:50,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:55,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:48:55,587 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:48:55,775 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:48:59,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:49:01,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:06,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:06,782 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:49:09,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:49:10,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:49:10,588 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:49:11,757 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:17,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:19,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:49:22,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:25,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:49:25,588 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:49:27,772 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:29,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:49:33,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:36,783 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:49:38,786 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:39,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:49:40,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:49:40,589 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:49:44,752 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:49,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:49:50,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:49:55,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:49:55,587 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:49:55,736 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:00,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:50:01,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:06,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:06,789 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:50:10,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:50:10,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:50:10,589 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:50:11,818 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:17,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:20,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:50:22,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:25,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:50:25,589 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:50:27,831 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:30,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:50:33,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:36,789 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:50:38,792 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:40,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:50:40,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:50:40,588 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:50:44,752 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:50,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:50:50,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:50:55,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:50:55,591 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:50:55,784 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:00,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:51:01,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:06,793 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:51:06,796 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:10,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:51:10,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:51:10,589 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:51:12,757 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:18,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:20,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:51:23,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:25,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:51:25,591 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:51:28,771 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:30,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:51:34,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:36,809 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:51:39,813 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:40,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:51:40,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:51:40,590 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:51:44,830 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:50,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:51:50,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:51:55,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:51:55,590 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:51:55,831 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:00,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:52:01,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:06,807 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:52:06,810 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:10,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:52:10,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:52:10,590 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:52:11,815 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:17,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:20,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:52:23,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:25,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:52:25,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:52:28,772 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:30,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:52:34,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:36,808 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:52:39,810 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:40,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:52:40,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:52:40,591 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:52:45,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:50,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:52:51,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:52:55,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:52:55,592 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:52:56,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:00,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:53:02,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:06,814 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:53:07,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:10,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:53:10,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:53:10,600 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:53:13,762 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:19,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:20,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:53:25,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:53:25,591 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:53:25,819 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:30,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:53:31,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:36,816 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:53:36,821 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:40,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:53:40,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:53:40,592 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:53:41,827 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:47,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:50,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:53:53,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:53:55,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:53:55,591 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:53:58,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:00,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:54:04,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:06,822 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:54:09,826 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:10,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:54:10,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:54:10,593 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:54:13,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:54:13,530 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:54:13,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:54:13,536 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:54:14,127 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:54:15,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:16,147 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:54:20,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:54:20,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:25,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:54:25,592 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:54:25,797 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:30,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:54:31,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:36,826 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:54:36,827 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:40,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:54:40,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:54:40,591 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:54:42,734 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:48,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:50,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:54:54,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:54:55,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:54:55,593 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:54:59,804 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:00,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:55:05,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:06,829 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:55:10,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:55:10,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:55:10,592 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:55:10,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:16,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:20,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:55:22,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:25,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:55:25,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:55:27,824 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:30,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:55:33,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:36,837 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:55:38,840 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:40,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:55:40,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:55:40,593 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:55:44,782 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:50,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:55:50,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:55:55,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:55:55,592 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:55:55,776 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:00,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:56:01,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:06,842 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:56:06,847 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:10,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:56:10,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:56:10,593 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:56:12,842 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:18,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:20,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:56:24,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:25,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:56:25,593 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:56:29,751 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:30,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:56:35,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:36,850 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:56:40,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:56:40,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:56:40,593 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:56:40,792 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:46,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:50,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:56:52,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:56:55,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:56:55,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:56:57,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:00,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:57:03,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:06,854 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:57:08,857 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:10,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:57:10,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:57:10,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:57:14,762 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:20,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:57:20,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:25,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:57:25,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:57:25,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:30,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:57:31,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:36,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:36,859 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:57:40,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:57:40,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:57:40,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:57:41,827 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:47,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:50,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:57:53,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:57:55,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:57:55,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:57:59,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:00,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:58:04,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:06,864 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:58:09,867 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:10,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:58:10,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:58:10,594 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:58:15,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:20,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:58:20,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:25,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:58:25,595 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:58:25,801 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:30,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:58:31,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:36,871 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:58:36,876 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:40,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:58:40,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:58:40,596 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:58:42,766 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:48,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:50,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:58:54,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:58:55,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:58:55,595 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:58:59,831 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:00,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:59:05,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:06,876 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:59:10,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:59:10,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:59:10,595 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:59:10,800 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:16,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:20,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:59:22,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:25,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:59:25,596 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:59:27,783 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:30,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:59:33,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:36,882 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 18:59:38,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:40,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:59:40,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:59:40,597 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:59:44,830 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:45,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 18:59:45,512 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 18:59:45,513 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 18:59:45,517 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 18:59:45,907 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 18:59:47,913 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 18:59:50,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 18:59:50,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 18:59:55,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 18:59:55,596 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 18:59:55,758 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:00,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:00:01,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:06,888 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:00:06,890 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:10,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:00:10,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:00:10,596 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:00:12,765 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:18,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:20,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:00:23,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:25,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:00:25,597 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:00:28,766 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:30,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:00:34,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:36,893 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:00:39,896 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:40,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:00:40,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:00:40,596 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:00:45,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:50,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:00:51,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:00:55,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:00:55,596 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:00:56,805 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:00,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:01:02,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:06,899 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:01:07,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:10,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:01:10,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:01:10,598 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:01:13,837 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:19,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:20,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:01:25,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:01:25,598 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:01:25,798 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:30,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:01:31,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:36,908 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:01:36,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:40,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:01:40,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:01:40,597 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:01:42,838 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:48,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:50,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:01:54,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:01:55,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:01:55,598 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:01:59,837 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:00,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:02:05,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:06,910 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:02:10,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:02:10,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:02:10,598 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:02:10,763 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:16,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:20,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:02:22,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:25,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:02:25,598 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:02:27,751 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:30,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:02:33,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:36,915 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:02:38,918 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:40,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:02:40,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:02:40,598 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:02:44,790 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:50,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:02:50,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:02:55,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:02:55,598 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:02:55,831 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:00,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:03:01,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:06,918 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:03:06,922 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:10,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:03:10,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:03:10,599 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:03:12,809 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:18,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:20,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:03:23,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:25,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:03:25,599 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:03:28,755 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:30,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:03:34,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:36,922 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:03:39,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:40,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:03:40,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:03:40,599 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:03:45,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:50,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:03:51,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:03:55,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:03:55,600 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:03:56,766 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:00,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:04:02,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:06,928 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:04:07,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:10,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:04:10,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:04:10,600 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:04:13,856 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:19,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:20,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:04:25,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:04:25,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:04:25,773 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:30,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:04:31,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:36,934 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:04:36,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:40,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:04:40,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:04:40,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:04:42,783 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:48,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:50,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:04:54,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:04:55,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:04:55,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:04:59,767 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:00,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:05:05,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:06,939 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:05:10,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:05:10,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:05:10,608 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:05:10,824 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:16,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:05:16,401 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:05:16,406 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:05:16,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:16,413 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:05:16,600 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:05:18,629 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:05:20,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:05:21,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:25,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:05:25,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:05:26,750 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:30,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:05:32,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:36,945 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:05:37,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:40,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:05:40,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:05:40,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:05:43,823 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:49,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:05:50,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:05:55,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:05:55,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:05:55,782 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:00,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:06:01,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:06,963 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:06:06,966 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:10,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:06:10,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:06:10,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:06:12,835 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:18,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:20,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:06:23,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:25,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:06:25,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:06:28,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:30,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:06:34,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:36,956 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:06:39,959 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:40,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:06:40,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:06:40,605 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:06:45,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:50,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:06:50,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:06:55,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:06:55,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:06:55,781 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:00,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:07:01,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:06,961 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:07:06,966 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:10,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:07:10,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:07:10,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:07:12,801 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:18,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:20,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:07:24,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:25,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:07:25,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:07:29,762 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:30,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:07:35,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:36,967 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:07:40,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:07:40,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:07:40,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:07:40,846 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:46,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:50,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:07:51,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:07:55,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:07:55,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:07:56,796 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:00,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:08:02,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:06,972 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:08:07,974 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:10,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:08:10,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:08:10,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:08:13,848 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:19,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:20,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:08:24,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:25,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:08:25,605 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:08:29,855 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:30,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:08:35,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:36,979 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:08:40,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:08:40,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:08:40,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:08:40,763 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:46,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:50,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:08:52,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:08:55,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:08:55,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:08:57,852 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:00,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:09:03,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:06,985 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:09:08,988 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:10,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:09:10,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:09:10,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:09:14,841 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:20,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:09:20,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:25,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:09:25,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:09:25,783 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:30,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:09:31,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:36,990 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:09:36,995 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:40,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:09:40,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:09:40,605 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:09:42,810 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:48,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:50,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:09:54,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:09:55,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:09:55,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:09:59,761 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:00,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:10:05,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:06,995 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:10:10,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:10:10,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:10:10,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:10:10,809 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:16,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:20,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:10:22,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:25,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:10:25,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:10:27,766 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:30,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:10:33,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:37,001 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:10:39,009 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:40,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:10:40,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:10:40,608 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:10:44,789 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:48,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:10:48,525 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:10:48,527 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:10:48,534 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:10:48,998 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:10:50,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:10:50,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:10:51,028 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:10:55,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:10:55,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:10:55,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:00,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:11:01,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:07,010 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:11:07,014 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:10,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:11:10,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:11:10,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:11:12,828 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:18,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:20,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:11:24,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:25,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:11:25,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:11:29,814 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:30,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:11:35,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:37,009 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:11:40,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:11:40,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:11:40,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:11:40,869 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:46,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:50,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:11:52,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:11:55,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:11:55,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:11:57,851 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:00,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:12:03,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:07,020 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:12:09,022 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:10,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:12:10,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:12:10,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:12:14,758 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:20,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:12:20,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:25,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:12:25,608 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:12:25,833 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:30,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:12:31,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:37,023 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:12:37,026 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:40,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:12:40,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:12:40,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:12:42,771 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:48,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:50,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:12:54,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:12:55,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:12:55,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:12:59,837 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:00,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:13:05,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:07,029 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:13:10,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:13:10,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:13:10,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:13:10,807 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:16,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:20,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:13:22,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:25,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:13:25,606 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:13:27,791 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:30,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:13:33,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:37,034 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:13:39,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:40,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:13:40,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:13:40,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:13:44,769 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:50,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:13:50,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:13:55,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:13:55,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:13:55,837 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:00,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:14:01,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:07,040 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:14:07,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:10,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:14:10,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:14:10,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:14:12,768 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:18,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:20,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:14:23,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:25,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:14:25,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:14:28,860 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:30,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:14:34,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:37,046 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:14:40,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:40,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:14:40,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:14:40,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:14:45,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:50,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:14:50,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:14:55,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:14:55,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:14:55,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:00,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:15:01,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:06,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:07,051 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:15:10,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:15:10,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:15:10,608 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:15:11,817 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:17,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:20,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:15:22,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:25,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:15:25,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:15:27,870 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:30,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:15:33,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:37,057 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:15:39,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:40,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:15:40,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:15:40,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:15:44,851 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:50,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:15:50,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:15:55,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:15:55,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:15:55,835 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:00,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:16:01,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:07,064 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:16:07,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:10,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:16:10,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:16:10,608 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:16:12,769 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:18,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:19,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:16:19,282 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:16:19,283 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:16:19,288 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:16:19,572 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:16:20,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:16:20,576 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:16:24,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:25,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:16:25,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:16:29,847 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:30,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:16:35,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:37,069 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:16:40,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:16:40,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:16:40,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:16:40,765 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:46,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:50,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:16:51,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:16:55,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:16:55,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:16:56,846 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:00,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:17:02,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:07,074 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:17:08,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:10,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:17:10,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:17:10,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:17:13,796 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:19,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:20,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:17:24,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:25,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:17:25,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:17:29,772 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:30,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:17:35,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:37,079 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:17:40,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:17:40,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:17:40,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:17:40,861 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:46,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:50,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:17:52,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:17:55,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:17:55,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:17:57,811 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:00,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:18:03,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:07,085 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:18:09,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:10,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:18:10,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:18:10,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:18:14,782 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:20,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:18:20,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:25,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:18:25,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:18:25,860 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:30,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:18:31,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:37,089 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:18:37,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:40,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:18:40,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:18:40,609 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:18:42,851 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:48,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:50,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:18:53,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:18:55,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:18:55,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:18:58,768 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:00,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:19:04,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:07,096 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:19:10,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:10,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:19:10,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:19:10,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:19:15,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:20,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:19:20,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:25,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:19:25,611 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:19:25,828 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:30,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:19:31,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:37,101 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:19:37,111 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:40,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:19:40,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:19:40,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:19:42,853 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:48,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:50,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:19:54,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:19:55,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:19:55,611 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:19:59,853 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:00,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:20:05,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:07,108 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:20:10,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:20:10,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:20:10,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:20:10,846 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:16,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:20,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:20:22,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:25,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:20:25,612 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:20:27,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:30,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:20:33,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:37,113 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:20:39,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:40,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:20:40,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:20:40,611 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:20:44,857 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:50,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:20:50,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:20:55,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:20:55,611 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:20:55,780 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:00,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:21:01,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:07,120 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:21:07,125 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:10,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:21:10,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:21:10,611 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:21:12,828 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:18,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:20,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:21:23,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:25,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:21:25,612 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:21:28,856 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:30,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:21:34,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:37,124 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:21:40,127 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:40,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:21:40,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:21:40,612 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:21:45,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:50,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:21:50,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:21:51,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:21:51,374 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:21:51,374 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:21:51,378 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:21:51,913 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:21:52,916 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:21:55,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:21:55,612 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:21:55,765 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:00,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:22:01,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:07,128 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:22:07,138 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:10,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:22:10,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:22:10,611 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:22:12,791 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:18,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:20,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:22:24,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:25,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:22:25,612 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:22:29,847 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:30,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:22:35,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:37,132 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:22:40,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:22:40,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:22:40,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:22:40,836 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:46,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:50,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:22:52,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:22:55,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:22:55,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:22:57,784 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:00,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:23:03,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:07,142 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:23:09,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:10,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:23:10,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:23:10,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:23:14,768 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:20,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:23:20,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:25,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:23:25,612 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:23:25,768 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:30,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:23:31,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:36,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:37,139 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:23:40,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:23:40,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:23:40,616 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:23:41,853 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:47,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:50,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:23:52,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:23:55,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:23:55,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:23:57,839 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:00,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:24:03,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:07,146 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:24:09,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:10,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:24:10,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:24:10,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:24:14,767 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:20,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:24:20,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:25,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:24:25,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:24:25,827 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:30,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:24:31,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:36,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:37,153 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:24:40,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:24:40,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:24:40,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:24:41,817 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:47,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:50,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:24:53,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:24:55,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:24:55,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:24:58,865 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:00,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:25:04,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:07,155 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:25:10,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:10,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:25:10,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:25:10,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:25:15,841 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:20,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:25:21,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:25,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:25:25,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:25:26,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:30,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:25:31,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:37,162 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:25:37,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:40,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:25:40,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:25:40,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:25:42,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:48,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:50,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:25:53,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:25:55,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:25:55,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:25:59,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:00,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:26:05,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:07,167 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:26:10,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:26:10,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:26:10,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:26:10,782 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:16,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:20,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:26:21,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:25,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:26:25,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:26:27,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:30,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:26:32,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:37,173 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:26:38,175 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:40,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:26:40,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:26:40,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:26:43,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:49,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:50,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:26:54,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:26:55,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:26:55,615 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:27:00,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:27:00,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:06,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:07,176 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:27:10,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:27:10,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:27:10,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:27:12,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:18,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:20,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:27:23,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:27:23,548 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:27:23,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:27:23,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:23,553 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:27:24,555 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:27:25,558 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:27:25,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:27:25,615 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:27:29,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:30,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:27:35,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:37,179 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:27:40,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:27:40,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:27:40,614 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:27:40,782 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:46,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:50,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:27:51,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:27:55,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:27:55,615 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:27:57,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:00,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:28:02,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:07,187 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:28:08,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:10,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:28:10,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:28:10,615 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:28:13,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:19,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:20,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:28:24,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:25,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:28:25,616 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:28:30,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:28:30,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:36,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:37,189 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:28:40,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:28:40,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:28:40,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:28:41,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:47,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:50,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:28:52,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:28:55,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:28:55,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:28:57,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:00,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:29:02,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:07,193 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:29:08,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:10,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:29:10,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:29:10,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:29:13,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:19,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:20,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:29:25,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:25,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:29:25,616 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:29:30,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:29:31,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:37,194 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:29:37,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:40,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:29:40,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:29:40,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:29:42,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:47,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:50,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:29:53,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:29:55,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:29:55,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:29:59,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:00,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:30:04,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:07,201 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:30:10,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:10,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:30:10,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:30:10,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:30:15,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:20,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:30:21,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:25,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:30:25,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:30:26,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:30,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:30:31,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:36,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:37,208 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:30:40,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:30:40,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:30:40,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:30:42,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:47,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:50,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:30:53,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:30:55,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:30:55,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:30:59,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:00,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:31:05,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:07,211 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:31:10,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:31:10,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:31:10,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:31:10,816 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:16,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:20,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:31:22,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:25,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:31:25,617 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:31:28,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:30,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:31:34,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:37,217 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:31:40,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:40,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:31:40,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:31:40,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:31:45,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:50,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:31:51,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:31:55,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:31:55,620 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:31:57,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:00,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:32:03,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:07,223 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:32:09,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:10,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:32:10,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:32:10,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:32:14,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:19,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:20,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:32:25,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:25,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:32:25,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:32:30,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:32:30,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:36,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:37,229 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:32:40,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:32:40,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:32:40,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:32:41,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:47,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:50,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:32:52,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:32:54,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:32:54,350 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:32:54,351 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:32:54,354 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:32:55,057 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:32:55,057 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:32:55,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:32:55,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:32:58,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:00,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:33:03,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:07,235 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:33:09,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:10,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:33:10,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:33:10,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:33:14,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:20,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:33:20,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:25,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:33:25,618 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:33:25,866 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:30,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:33:31,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:36,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:37,240 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:33:40,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:33:40,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:33:40,619 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:33:42,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:47,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:50,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:33:53,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:33:55,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:33:55,619 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:33:59,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:00,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:34:04,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:07,247 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:34:10,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:34:10,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:10,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:34:10,619 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:34:15,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:20,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:34:21,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:25,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:34:25,619 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:34:27,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:30,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:34:32,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:37,251 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:34:38,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:40,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:34:40,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:34:40,619 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:34:43,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:49,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:50,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:34:55,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:34:55,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:34:55,619 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:35:00,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:35:01,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:06,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:07,257 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:35:10,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:35:10,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:35:10,620 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:35:12,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:18,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:20,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:35:24,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:25,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:35:25,620 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:35:29,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:30,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:35:35,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:37,263 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:35:40,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:35:40,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:35:40,620 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:35:40,854 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:46,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:50,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:35:51,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:35:55,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:35:55,620 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:35:57,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:00,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:36:02,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:07,266 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:36:08,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:10,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:36:10,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:36:10,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:36:13,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:19,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:20,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:36:25,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:25,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:36:25,620 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:36:30,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:36:30,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:36,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:37,273 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:36:40,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:36:40,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:36:40,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:36:42,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:48,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:50,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:36:54,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:36:55,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:36:55,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:37:00,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:37:00,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:06,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:07,278 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:37:10,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:37:10,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:37:10,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:37:12,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:18,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:20,230 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:37:24,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:25,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:37:25,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:37:29,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:30,230 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:37:34,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:37,283 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:37:40,230 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:37:40,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:40,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:37:40,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:37:45,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:50,230 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:37:51,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:37:55,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:37:55,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:37:56,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:00,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:38:02,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:07,290 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:38:08,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:10,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:38:10,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:38:10,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:38:13,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:18,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:20,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:38:24,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:25,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:38:25,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:38:25,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:38:25,946 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:38:25,947 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:38:25,950 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:38:26,261 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:38:27,265 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:38:29,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:30,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:38:35,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:37,296 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:38:40,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:38:40,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:40,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:38:40,621 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:38:46,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:50,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:38:52,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:38:55,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:38:55,622 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:38:58,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:00,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:39:03,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:07,301 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:39:08,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:10,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:39:10,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:39:10,623 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:39:14,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:20,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:39:20,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:25,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:25,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:39:25,622 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:39:30,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:39:30,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:36,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:37,307 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:39:40,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:39:40,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:39:40,624 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:39:42,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:48,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:50,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:39:53,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:39:55,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:39:55,623 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:39:59,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:00,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:40:04,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:07,314 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:40:09,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:10,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:40:10,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:40:10,623 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:40:15,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:20,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:40:21,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:25,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:40:25,623 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:40:27,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:30,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:40:32,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:37,318 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:40:37,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:40,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:40:40,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:40:40,623 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:40:43,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:49,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:50,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:40:54,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:40:55,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:40:55,624 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:41:00,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:41:00,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:06,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:07,324 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:41:10,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:41:10,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:41:10,623 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:41:12,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:17,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:20,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:41:23,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:25,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:41:25,624 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:41:29,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:30,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:41:35,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:37,330 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:41:40,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:41:40,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:40,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:41:40,623 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:41:46,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:50,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:41:51,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:41:55,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:41:55,624 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:41:57,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:00,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:42:03,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:07,336 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:42:08,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:10,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:42:10,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:42:10,624 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:42:14,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:20,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:42:20,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:25,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:42:25,624 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:42:25,799 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:30,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:42:31,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:37,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:37,343 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:42:40,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:42:40,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:42:40,626 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:42:43,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:49,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:50,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:42:54,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:42:55,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:42:55,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:43:00,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:43:00,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:06,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:07,347 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:43:10,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:43:10,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:43:10,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:43:11,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:16,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:20,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:43:22,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:25,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:43:25,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:43:28,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:30,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:43:34,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:37,353 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:43:39,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:40,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:43:40,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:43:40,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:43:45,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:50,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:43:51,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:55,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:43:55,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:43:57,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:43:57,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:43:57,713 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:43:57,714 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:43:57,718 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:43:58,570 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:43:59,574 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:44:00,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:44:03,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:07,359 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:44:08,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:10,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:44:10,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:44:10,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:44:14,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:19,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:20,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:44:25,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:25,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:44:25,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:44:30,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:44:31,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:37,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:37,364 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:44:40,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:44:40,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:44:40,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:44:43,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:49,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:50,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:44:55,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:44:55,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:44:55,626 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:45:00,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:45:00,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:06,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:07,373 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:45:10,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:45:10,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:45:10,625 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:45:12,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:18,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:20,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:45:24,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:25,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:45:25,626 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:45:30,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:45:30,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:36,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:37,371 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:45:40,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:45:40,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:45:40,626 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:45:42,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:48,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:50,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:45:54,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:45:55,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:45:55,626 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:45:59,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:00,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:46:05,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:07,381 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:46:10,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:46:10,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:10,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:46:10,626 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:46:16,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:20,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:46:22,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:25,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:46:25,627 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:46:27,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:30,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:46:33,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:37,383 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:46:38,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:40,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:46:40,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:46:40,627 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:46:44,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:50,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:46:50,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:46:55,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:46:55,627 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:46:55,875 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:00,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:47:01,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:06,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:07,389 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:47:10,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:47:10,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:47:10,627 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:47:12,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:18,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:20,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:47:23,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:25,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:47:25,627 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:47:29,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:30,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:47:35,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:37,395 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:47:40,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:47:40,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:40,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:47:40,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:47:46,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:50,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:47:51,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:47:55,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:47:55,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:47:57,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:00,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:48:03,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:07,399 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:48:08,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:10,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:48:10,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:48:10,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:48:14,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:20,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:48:20,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:25,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:25,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:48:25,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:48:30,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:48:30,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:36,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:37,407 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:48:40,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:48:40,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:48:40,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:48:42,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:48,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:50,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:48:53,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:48:55,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:48:55,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:48:59,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:00,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:49:05,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:07,411 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:49:10,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:49:10,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:10,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:49:10,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:49:16,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:20,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:49:22,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:25,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:49:25,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:49:28,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:49:28,289 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:49:28,290 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:49:28,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:28,294 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:49:28,919 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:49:29,922 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:49:30,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:49:33,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:37,417 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:49:38,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:40,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:49:40,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:49:40,629 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:49:44,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:50,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:49:50,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:55,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:49:55,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:49:55,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:50:00,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:50:01,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:07,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:07,425 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:50:10,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:50:10,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:50:10,629 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:50:13,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:18,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:20,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:50:24,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:25,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:50:25,629 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:50:29,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:30,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:50:35,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:37,428 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:50:40,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:50:40,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:40,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:50:40,629 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:50:46,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:50,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:50:51,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:50:55,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:50:55,629 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:50:57,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:00,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:51:03,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:07,434 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:51:08,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:10,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:51:10,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:51:10,629 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:51:14,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:20,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:51:20,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:25,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:51:25,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:51:25,859 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:30,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:51:31,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:36,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:37,439 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:51:40,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:51:40,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:51:40,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:51:42,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:48,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:50,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:51:54,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:51:55,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:51:55,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:52:00,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:52:00,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:06,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:07,441 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:52:10,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:52:10,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:52:10,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:52:12,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:18,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:20,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:52:23,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:25,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:52:25,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:52:29,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:30,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:52:35,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:37,444 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:52:40,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:52:40,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:40,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:52:40,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:52:46,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:50,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:52:51,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:52:55,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:52:55,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:52:57,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:00,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:53:02,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:07,445 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:53:07,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:10,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:53:10,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:53:10,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:53:13,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:19,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:20,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:53:25,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:25,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:53:25,631 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:53:30,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:53:31,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:37,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:37,452 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:53:40,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:53:40,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:53:40,632 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:53:43,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:48,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:50,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:53:54,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:53:55,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:53:55,631 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:54:00,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:54:00,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:06,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:07,454 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:54:10,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:54:10,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:54:10,631 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:54:12,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:18,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:20,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:54:24,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:25,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:54:25,631 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:54:30,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:54:30,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:36,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:37,456 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:54:40,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:54:40,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:54:40,632 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:54:42,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:48,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:50,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:54:54,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:55,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:54:55,633 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:54:59,959 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 19:54:59,962 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 19:54:59,963 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 19:54:59,963 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:54:59,967 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 19:55:00,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:55:00,269 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 19:55:02,276 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 19:55:05,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:07,459 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:55:10,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:55:10,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:10,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:55:10,631 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:55:16,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:20,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:55:22,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:25,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:55:25,631 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:55:28,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:30,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:55:34,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:37,464 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:55:39,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:40,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:55:40,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:55:40,632 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:55:45,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:50,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:55:50,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:55:55,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:55:55,632 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:55:55,866 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:00,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:56:01,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:07,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:07,467 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:56:10,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:56:10,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:56:10,632 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:56:12,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:18,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:20,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:56:23,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:25,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:56:25,632 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:56:29,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:30,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:56:34,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:37,471 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:56:39,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:40,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:56:40,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:56:40,633 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:56:45,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:50,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:56:51,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:56:55,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:56:55,633 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:56:56,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:00,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:57:02,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:07,477 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:57:07,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:10,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:57:10,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:57:10,633 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:57:13,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:19,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:20,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:57:24,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:25,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:57:25,632 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:57:30,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:57:30,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:36,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:37,476 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:57:40,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:57:40,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:57:40,633 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:57:42,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:47,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:50,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:57:52,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:57:55,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:57:55,791 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:57:58,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:00,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:58:04,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:07,487 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:58:09,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:10,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:58:10,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:58:10,633 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:58:15,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:20,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:58:20,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:25,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:58:25,633 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:58:25,781 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:30,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:58:31,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:37,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:37,492 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:58:40,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:58:40,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:58:40,634 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:58:43,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:48,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:50,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:58:54,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:58:55,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:58:55,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:58:59,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:00,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:59:05,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:07,497 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:59:10,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:59:10,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:10,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:59:10,634 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:59:16,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:20,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:59:22,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:25,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:59:25,634 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:59:28,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:30,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:59:34,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:37,508 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 19:59:39,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:40,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:59:40,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:59:40,634 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:59:45,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:50,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 19:59:50,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 19:59:55,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 19:59:55,635 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 19:59:55,877 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:00,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:00:01,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:07,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:07,508 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:00:10,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:00:10,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:00:10,635 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:00:12,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:18,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:20,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:00:24,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:25,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:00:25,634 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:00:30,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:00:30,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:31,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:00:31,076 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:00:31,077 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:00:31,081 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:00:31,741 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:00:32,745 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:00:36,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:37,514 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:00:40,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:00:40,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:00:40,635 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:00:42,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:47,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:50,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:00:53,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:00:55,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:00:55,635 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:00:58,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:00,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:01:04,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:07,520 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:01:09,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:10,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:01:10,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:01:10,635 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:01:15,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:20,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:01:21,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:25,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:01:25,635 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:01:27,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:30,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:01:33,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:37,524 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:01:38,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:40,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:01:40,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:01:40,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:01:44,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:50,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:01:50,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:55,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:01:55,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:01:55,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:02:00,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:02:01,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:07,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:07,530 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:02:10,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:02:10,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:02:10,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:02:12,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:18,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:20,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:02:24,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:25,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:02:25,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:02:30,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:02:30,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:35,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:37,532 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:02:40,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:02:40,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:40,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:02:40,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:02:46,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:50,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:02:51,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:02:55,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:02:55,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:02:57,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:00,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:03:03,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:07,535 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:03:08,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:10,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:03:10,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:03:10,636 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:03:14,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:19,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:20,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:03:25,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:25,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:03:25,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:03:30,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:03:31,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:36,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:37,537 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:03:40,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:03:40,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:03:40,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:03:42,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:47,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:50,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:03:53,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:03:55,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:03:55,637 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:03:59,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:00,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:04:04,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:07,543 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:04:09,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:10,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:04:10,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:04:10,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:04:15,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:20,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:04:21,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:25,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:04:25,637 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:04:27,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:30,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:04:33,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:37,549 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:04:38,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:40,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:04:40,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:04:40,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:04:44,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:50,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:04:50,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:04:55,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:04:55,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:04:55,790 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:00,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:05:01,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:07,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:07,553 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:05:10,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:05:10,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:05:10,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:05:12,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:18,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:20,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:05:24,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:25,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:05:25,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:05:30,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:05:30,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:35,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:37,559 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:05:40,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:05:40,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:40,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:05:40,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:05:46,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:50,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:05:52,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:05:55,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:05:55,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:05:58,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:00,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:06:02,747 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:06:02,750 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:06:02,751 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:06:02,754 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:06:03,116 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:06:03,791 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:04,119 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:06:07,565 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:06:09,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:10,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:06:10,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:06:10,638 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:06:15,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:20,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:06:20,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:25,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:06:25,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:06:25,841 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:30,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:06:31,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:37,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:37,570 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:06:40,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:06:40,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:06:40,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:06:42,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:48,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:50,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:06:54,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:06:55,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:06:55,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:06:59,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:00,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:07:04,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:07,577 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:07:09,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:10,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:07:10,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:07:10,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:07:15,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:20,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:07:20,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:25,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:07:25,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:07:25,862 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:30,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:07:31,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:37,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:37,582 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:07:40,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:07:40,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:07:40,640 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:07:43,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:48,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:50,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:07:53,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:07:55,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:07:55,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:07:59,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:00,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:08:05,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:07,587 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:08:10,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:08:10,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:10,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:08:10,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:08:16,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:20,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:08:22,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:25,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:08:25,640 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:08:28,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:30,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:08:34,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:37,592 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:08:39,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:40,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:08:40,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:08:40,640 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:08:45,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:50,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:08:50,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:08:55,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:08:55,639 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:08:55,795 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:00,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:09:01,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:06,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:07,598 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:09:10,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:09:10,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:09:10,640 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:09:12,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:18,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:20,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:09:23,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:25,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:09:25,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:09:29,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:30,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:09:35,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:37,608 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:09:40,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:09:40,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:40,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:09:40,640 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:09:46,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:50,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:09:51,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:09:55,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:09:55,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:09:56,855 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:00,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:10:02,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:07,609 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:10:07,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:10,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:10:10,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:10:10,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:10:13,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:19,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:20,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:10:25,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:25,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:10:25,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:10:30,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:10:31,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:37,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:37,613 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:10:40,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:10:40,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:10:40,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:10:43,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:49,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:50,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:10:55,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:10:55,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:10:55,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:11:00,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:11:01,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:07,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:07,618 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:11:10,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:11:10,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:11:10,642 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:11:12,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:18,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:20,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:11:24,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:25,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:11:25,641 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:11:30,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:11:30,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:34,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:11:34,611 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:11:34,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:11:34,625 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:11:35,416 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:11:35,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:36,429 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:11:37,623 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:11:40,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:11:40,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:11:40,642 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:11:40,868 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:46,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:50,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:11:52,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:11:55,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:11:55,642 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:11:58,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:00,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:12:03,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:07,630 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:12:08,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:10,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:12:10,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:12:10,642 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:12:14,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:20,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:12:20,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:25,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:12:25,642 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:12:25,806 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:30,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:12:31,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:36,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:37,635 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:12:40,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:12:40,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:12:40,642 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:12:41,844 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:47,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:50,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:12:53,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:12:55,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:12:55,643 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:12:59,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:00,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:13:05,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:07,641 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:13:10,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:13:10,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:13:10,643 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:13:10,848 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:16,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:20,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:13:22,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:25,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:13:25,643 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:13:28,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:30,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:13:34,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:37,645 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:13:39,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:40,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:13:40,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:13:40,644 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:13:45,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:50,263 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:13:51,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:13:55,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:13:55,644 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:13:56,810 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:00,263 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:14:02,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:07,653 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:14:07,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:10,263 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:14:10,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:14:10,644 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:14:13,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:19,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:20,263 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:14:25,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:25,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:14:25,643 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:14:30,263 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:14:31,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:36,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:37,659 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:14:40,263 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:14:40,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:14:40,644 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:14:41,887 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:47,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:50,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:14:53,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:14:55,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:14:55,644 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:14:59,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:00,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:15:05,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:07,664 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:15:10,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:15:10,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:15:10,645 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:15:10,824 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:16,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:20,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:15:22,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:25,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:15:25,645 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:15:27,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:30,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:15:33,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:37,667 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:15:38,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:40,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:15:40,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:15:40,645 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:15:44,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:49,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:50,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:15:55,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:15:55,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:15:55,645 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:16:00,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:16:01,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:07,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:07,675 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:16:10,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:16:10,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:16:10,645 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:16:13,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:19,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:20,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:16:25,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:25,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:16:25,646 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:16:30,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:16:31,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:37,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:37,677 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:16:40,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:16:40,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:16:40,646 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:16:43,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:48,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:50,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:16:54,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:16:55,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:16:55,646 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:16:59,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:00,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:17:05,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:05,731 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:17:05,735 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:17:05,736 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:17:05,739 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:17:06,677 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:17:06,678 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:17:07,684 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:17:10,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:17:10,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:17:10,646 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:17:10,894 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:16,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:20,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:17:22,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:25,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:17:25,647 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:17:28,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:30,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:17:33,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:37,689 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:17:38,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:40,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:17:40,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:17:40,646 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:17:44,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:49,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:50,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:17:55,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:17:55,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:17:55,646 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:18:00,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:18:00,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:06,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:07,695 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:18:10,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:18:10,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:18:10,647 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:18:11,868 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:17,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:20,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:18:23,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:25,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:18:25,646 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:18:29,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:30,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:18:35,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:37,701 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:18:40,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:18:40,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:18:40,647 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:18:40,866 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:46,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:50,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:18:52,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:18:55,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:18:55,647 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:18:57,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:00,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:19:02,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:07,708 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:19:07,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:10,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:19:10,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:19:10,647 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:19:13,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:18,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:20,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:19:24,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:25,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:19:25,647 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:19:30,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:19:30,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:36,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:37,713 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:19:40,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:19:40,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:19:40,648 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:19:41,826 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:47,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:50,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:19:53,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:19:55,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:19:55,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:19:58,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:00,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:20:04,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:07,718 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:20:09,721 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:10,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:20:10,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:20:10,648 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:20:15,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:20,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:20:21,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:25,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:20:25,648 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:20:26,833 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:30,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:20:32,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:37,725 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:20:37,730 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:40,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:20:40,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:20:40,648 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:20:43,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:49,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:50,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:20:54,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:20:55,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:20:55,649 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:21:00,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:21:00,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:06,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:07,729 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:21:10,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:21:10,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:21:10,648 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:21:11,878 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:17,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:20,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:21:22,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:25,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:21:25,649 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:21:27,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:30,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:21:33,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:37,735 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:21:38,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:40,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:21:40,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:21:40,649 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:21:44,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:50,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:21:50,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:21:55,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:21:55,649 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:21:55,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:00,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:22:01,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:07,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:07,741 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:22:10,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:22:10,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:22:10,650 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:22:13,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:19,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:20,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:22:24,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:25,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:22:25,650 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:22:30,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:22:30,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:36,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:37,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:22:37,609 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:22:37,610 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:22:37,617 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:22:37,747 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:22:37,933 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:22:38,937 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:22:40,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:22:40,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:22:40,650 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:22:41,895 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:47,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:50,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:22:53,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:22:55,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:22:55,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:22:59,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:00,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:23:05,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:07,753 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:23:10,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:23:10,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:23:10,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:23:10,861 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:16,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:20,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:23:21,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:25,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:23:25,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:23:26,870 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:30,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:23:32,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:37,759 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:23:37,764 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:40,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:23:40,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:23:40,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:23:43,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:48,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:50,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:23:54,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:23:55,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:23:55,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:24:00,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:24:00,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:06,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:07,765 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:24:10,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:24:10,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:24:10,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:24:11,865 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:17,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:20,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:24:23,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:25,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:24:25,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:24:29,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:30,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:24:34,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:37,768 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:24:39,773 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:40,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:24:40,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:24:40,651 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:24:45,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:50,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:24:50,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:24:55,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:24:55,652 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:24:55,831 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:00,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:25:01,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:06,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:07,774 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:25:10,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:25:10,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:25:10,652 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:25:11,892 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:17,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:20,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:25:22,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:25,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:25:25,652 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:25:27,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:30,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:25:33,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:37,778 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:25:38,780 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:40,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:25:40,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:25:40,652 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:25:44,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:50,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:25:50,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:55,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:25:55,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:25:55,652 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:26:00,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:26:01,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:07,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:07,783 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:26:10,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:26:10,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:26:10,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:26:13,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:18,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:20,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:26:24,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:25,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:26:25,652 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:26:29,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:30,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:26:35,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:37,785 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:26:40,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:26:40,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:26:40,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:26:40,875 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:46,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:50,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:26:52,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:26:55,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:26:55,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:26:58,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:00,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:27:03,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:07,791 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:27:08,792 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:10,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:27:10,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:27:10,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:27:14,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:20,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:27:20,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:25,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:27:25,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:27:25,804 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:30,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:27:31,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:37,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:37,796 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:27:40,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:27:40,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:27:40,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:27:42,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:48,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:50,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:27:54,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:27:55,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:27:55,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:28:00,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:28:00,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:05,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:07,801 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:28:09,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:28:09,558 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:28:09,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:28:09,563 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:28:10,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:28:10,322 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:28:10,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:10,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:28:10,653 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:28:11,324 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:28:16,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:20,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:28:22,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:25,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:28:25,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:28:28,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:30,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:28:34,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:37,808 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:28:39,811 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:40,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:28:40,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:28:40,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:28:45,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:50,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:28:51,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:28:55,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:28:55,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:28:56,862 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:00,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:29:02,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:07,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:07,815 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:29:10,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:29:10,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:29:10,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:29:13,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:19,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:20,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:29:24,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:25,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:29:25,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:29:30,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:29:30,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:36,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:37,819 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:29:40,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:29:40,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:29:40,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:29:41,892 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:47,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:50,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:29:53,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:29:55,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:29:55,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:29:59,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:00,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:30:04,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:07,823 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:30:09,826 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:10,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:30:10,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:30:10,655 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:30:15,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:20,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:30:20,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:25,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:30:25,654 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:30:25,835 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:30,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:30:31,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:36,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:37,831 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:30:40,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:30:40,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:30:40,655 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:30:41,878 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:47,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:50,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:30:53,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:30:55,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:30:55,655 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:30:58,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:00,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:31:04,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:07,836 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:31:09,838 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:10,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:31:10,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:31:10,655 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:31:15,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:20,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:31:21,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:25,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:31:25,658 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:31:26,819 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:30,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:31:32,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:37,842 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:31:37,847 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:40,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:31:40,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:31:40,656 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:31:43,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:48,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:50,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:31:54,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:31:55,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:31:55,656 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:32:00,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:32:00,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:06,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:07,847 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:32:10,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:32:10,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:32:10,656 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:32:11,875 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:17,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:20,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:32:23,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:25,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:32:25,657 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:32:29,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:30,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:32:35,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:37,852 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:32:40,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:32:40,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:32:40,656 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:32:40,836 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:46,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:50,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:32:52,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:32:55,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:32:55,656 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:32:57,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:00,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:33:02,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:07,859 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:33:07,864 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:10,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:33:10,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:33:10,656 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:33:13,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:18,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:20,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:33:24,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:25,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:33:25,657 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:33:29,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:30,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:33:35,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:37,861 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:33:40,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:33:40,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:33:40,673 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:33:40,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:33:40,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:40,678 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:33:40,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:33:40,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:33:40,769 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:33:41,781 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:33:46,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:50,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:33:52,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:33:55,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:33:55,657 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:33:58,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:00,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:34:04,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:07,869 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:34:09,871 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:10,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:34:10,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:34:10,657 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:34:15,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:20,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:34:21,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:25,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:34:25,657 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:34:26,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:30,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:34:32,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:37,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:37,873 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:34:40,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:34:40,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:34:40,658 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:34:43,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:48,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:50,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:34:54,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:34:55,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:34:55,657 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:35:00,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:35:00,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:06,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:07,878 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:35:10,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:35:10,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:35:10,657 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:35:11,898 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:17,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:20,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:35:22,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:25,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:35:25,658 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:35:28,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:30,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:35:34,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:37,883 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:35:39,886 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:40,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:35:40,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:35:40,658 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:35:45,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:50,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:35:51,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:35:55,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:35:55,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:35:56,833 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:00,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:36:02,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:07,888 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:36:07,893 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:10,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:36:10,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:36:10,658 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:36:13,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:18,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:20,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:36:24,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:25,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:36:25,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:36:30,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:36:30,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:36,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:37,894 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:36:40,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:36:40,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:36:40,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:36:41,825 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:47,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:50,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:36:53,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:36:55,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:36:55,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:36:59,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:00,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:37:05,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:07,901 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:37:10,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:37:10,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:37:10,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:37:10,857 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:16,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:20,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:37:22,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:25,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:37:25,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:37:27,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:30,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:37:32,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:37,907 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:37:37,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:40,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:37:40,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:37:40,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:37:43,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:49,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:50,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:37:54,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:37:55,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:37:55,660 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:38:00,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:38:00,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:05,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:07,913 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:38:10,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:38:10,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:38:10,660 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:38:10,895 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:16,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:20,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:38:21,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:25,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:38:25,659 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:38:26,868 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:30,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:38:32,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:37,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:37,917 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:38:40,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:38:40,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:38:40,660 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:38:43,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:49,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:50,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:38:55,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:38:55,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:38:55,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:39:00,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:39:01,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:06,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:07,923 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:39:10,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:39:10,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:39:10,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:39:11,881 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:13,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:39:13,069 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:39:13,070 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:39:13,081 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:39:13,090 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:39:16,124 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:39:17,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:20,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:39:22,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:25,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:39:25,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:39:28,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:30,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:39:34,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:37,925 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:39:39,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:40,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:39:40,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:39:40,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:39:45,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:50,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:39:51,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:39:55,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:39:55,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:39:56,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:00,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:40:02,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:07,929 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:40:07,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:10,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:40:10,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:40:10,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:40:13,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:18,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:20,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:40:24,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:25,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:40:25,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:40:30,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:40:30,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:35,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:37,934 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:40:40,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:40:40,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:40:40,662 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:40:40,891 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:46,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:50,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:40:51,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:40:55,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:40:55,662 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:40:56,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:00,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:41:02,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:07,940 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:41:07,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:10,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:41:10,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:41:10,661 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:41:13,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:19,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:20,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:41:25,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:25,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:41:25,662 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:41:30,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:41:30,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:36,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:37,947 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:41:40,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:41:40,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:41:40,662 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:41:41,823 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:47,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:50,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:41:52,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:41:55,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:41:55,663 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:41:58,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:00,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:42:03,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:07,951 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:42:08,953 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:10,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:42:10,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:42:10,663 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:42:14,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:20,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:42:20,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:25,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:42:25,663 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:42:25,871 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:30,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:42:31,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:37,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:37,958 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:42:40,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:42:40,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:42:40,663 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:42:43,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:49,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:50,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:42:55,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:42:55,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:42:55,663 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:43:00,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:43:01,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:07,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:07,963 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:43:10,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:43:10,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:43:10,663 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:43:13,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:18,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:20,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:43:24,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:25,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:43:25,663 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:43:30,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:43:30,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:36,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:37,972 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:43:40,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:43:40,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:43:40,664 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:43:41,830 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:47,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:50,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:43:52,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:43:55,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:43:55,664 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:43:57,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:00,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:44:03,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:07,971 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:44:08,974 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:10,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:44:10,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:44:10,665 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:44:14,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:20,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:44:20,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:25,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:44:25,664 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:44:25,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:30,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:44:31,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:37,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:37,976 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:44:40,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:44:40,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:44:40,664 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:44:43,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:45,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:44:45,669 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:44:45,670 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:44:45,673 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:44:46,545 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:44:47,548 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:44:49,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:50,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:44:55,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:44:55,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:44:55,665 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:45:00,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:45:01,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:07,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:07,979 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:45:10,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:45:10,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:45:10,665 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:45:12,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:18,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:20,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:45:23,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:25,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:45:25,664 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:45:29,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:30,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:45:35,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:37,985 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:45:40,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:45:40,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:45:40,665 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:45:40,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:46,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:50,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:45:52,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:45:55,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:45:55,665 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:45:58,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:00,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:46:04,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:07,990 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:46:09,992 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:10,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:46:10,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:46:10,665 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:46:15,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:20,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:46:20,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:25,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:25,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:46:25,670 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:46:30,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:46:31,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:36,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:37,995 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:46:40,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:46:40,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:46:40,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:46:41,838 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:47,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:50,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:46:52,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:46:55,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:46:55,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:46:58,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:00,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:47:03,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:08,002 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:47:09,007 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:10,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:47:10,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:47:10,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:47:14,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:20,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:47:20,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:25,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:47:25,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:47:25,898 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:30,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:47:31,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:36,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:38,007 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:47:40,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:47:40,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:47:40,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:47:41,890 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:47,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:50,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:47:53,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:47:55,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:47:55,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:47:59,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:00,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:48:05,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:08,013 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:48:10,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:48:10,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:48:10,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:48:10,881 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:16,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:20,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:48:21,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:25,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:48:25,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:48:26,860 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:30,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:48:32,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:38,019 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:48:38,025 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:40,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:48:40,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:48:40,671 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:48:43,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:49,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:50,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:48:54,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:48:55,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:48:55,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:49:00,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:49:00,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:06,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:08,020 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:49:10,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:49:10,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:49:10,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:49:11,857 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:17,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:20,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:49:22,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:25,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:49:25,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:49:28,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:30,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:49:34,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:38,028 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:49:40,031 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:40,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:49:40,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:49:40,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:49:45,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:50,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:49:51,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:49:55,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:49:55,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:49:56,811 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:00,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:50:02,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:08,032 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:50:08,037 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:10,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:50:10,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:50:10,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:50:13,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:16,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:50:16,815 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:50:16,816 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:50:16,819 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:50:17,746 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:50:17,747 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:50:19,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:20,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:50:24,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:25,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:50:25,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:50:30,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:50:30,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:36,519 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:38,038 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:50:40,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:50:40,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:50:40,672 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:50:41,872 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:47,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:50,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:50:52,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:50:55,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:50:55,673 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:50:58,519 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:00,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:51:04,519 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:08,043 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:51:10,046 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:10,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:51:10,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:51:10,673 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:51:15,519 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:20,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:51:21,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:25,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:51:25,673 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:51:26,895 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:30,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:51:32,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:38,050 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:51:38,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:40,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:51:40,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:51:40,673 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:51:43,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:49,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:50,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:51:55,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:51:55,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:51:55,673 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:52:00,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:52:01,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:06,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:08,054 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:52:10,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:52:10,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:52:10,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:52:11,874 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:17,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:20,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:52:22,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:25,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:52:25,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:52:27,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:30,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:52:33,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:38,060 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:52:39,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:40,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:52:40,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:52:40,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:52:44,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:50,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:52:50,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:55,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:52:55,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:52:55,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:53:00,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:53:01,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:06,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:08,067 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:53:10,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:53:10,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:53:10,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:53:11,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:17,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:20,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:53:22,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:25,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:53:25,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:53:28,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:30,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:53:34,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:38,068 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:53:40,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:40,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:53:40,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:53:40,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:53:45,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:50,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:53:50,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:53:55,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:53:55,675 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:53:55,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:00,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:54:01,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:07,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:08,074 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:54:10,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:54:10,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:54:10,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:54:13,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:19,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:20,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:54:24,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:25,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:54:25,675 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:54:30,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:54:30,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:36,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:38,080 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:54:40,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:54:40,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:54:40,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:54:41,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:47,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:50,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:54:53,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:54:55,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:54:55,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:54:59,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:00,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:55:05,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:08,086 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:55:10,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:55:10,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:55:10,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:55:10,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:16,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:20,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:55:22,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:25,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:55:25,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:55:28,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:30,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:55:34,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:38,087 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:55:40,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:40,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:55:40,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:55:40,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:55:45,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:49,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 20:55:49,240 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 20:55:49,241 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 20:55:49,244 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 20:55:50,041 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 20:55:50,043 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 20:55:50,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:55:51,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:55:55,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:55:55,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:55:56,843 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:00,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:56:02,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:07,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:08,089 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:56:10,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:56:10,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:56:10,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:56:13,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:19,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:20,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:56:24,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:25,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:56:25,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:56:30,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:56:30,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:36,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:38,128 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:56:40,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:56:40,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:56:40,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:56:41,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:47,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:50,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:56:52,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:56:55,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:56:55,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:56:58,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:00,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:57:04,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:08,136 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:57:10,139 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:10,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:57:10,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:57:10,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:57:15,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:20,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:57:20,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:25,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:57:25,676 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:57:26,012 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:30,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:57:31,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:37,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:38,140 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:57:40,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:57:40,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:57:40,677 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:57:42,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:47,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:50,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:57:53,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:57:55,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:57:55,677 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:57:58,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:00,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:58:04,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:08,146 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:58:10,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:10,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:58:10,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:58:10,677 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:58:15,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:20,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:58:21,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:25,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:58:25,677 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:58:26,834 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:30,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:58:32,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:38,152 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:58:38,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:40,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:58:40,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:58:40,677 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:58:43,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:48,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:50,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:58:54,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:58:55,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:58:55,679 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:59:00,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:59:00,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:05,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:08,156 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:59:10,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:59:10,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:59:10,677 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:59:10,884 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:16,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:20,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:59:22,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:25,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:59:25,678 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:59:27,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:30,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:59:33,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:38,163 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 20:59:39,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:40,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:59:40,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:59:40,678 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:59:44,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:50,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 20:59:50,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 20:59:55,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 20:59:55,678 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 20:59:55,922 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:00,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:00:01,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:07,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:08,169 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:00:10,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:00:10,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:00:10,678 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:00:12,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:18,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:20,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:00:24,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:25,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:00:25,678 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:00:29,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:30,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:00:35,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:38,174 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:00:40,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:00:40,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:00:40,678 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:00:40,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:46,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:50,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:00:52,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:00:55,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:00:55,679 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:00:58,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:00,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:01:04,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:08,180 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:01:10,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:10,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:01:10,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:01:10,679 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:01:15,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:20,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:01:20,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:01:20,511 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:01:20,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:01:20,515 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:01:20,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:21,293 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:01:22,297 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:01:25,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:01:25,679 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:01:25,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:30,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:01:31,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:37,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:38,187 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:01:40,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:01:40,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:01:40,679 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:01:43,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:49,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:50,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:01:55,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:01:55,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:01:55,679 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:02:00,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:02:01,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:07,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:08,192 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:02:10,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:02:10,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:02:10,680 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:02:12,863 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:18,021 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:20,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:02:23,022 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:25,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:02:25,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:02:29,021 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:30,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:02:34,022 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:38,199 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:02:39,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:40,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:02:40,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:02:40,680 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:02:45,023 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:50,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:02:51,023 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:02:55,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:02:55,680 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:02:56,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:00,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:03:02,023 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:08,023 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:08,211 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:03:10,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:03:10,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:03:10,680 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:03:14,023 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:19,023 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:20,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:03:24,024 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:25,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:03:25,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:03:30,024 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:30,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:03:36,024 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:38,210 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:03:40,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:03:40,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:03:40,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:03:41,836 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:47,024 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:50,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:03:53,025 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:03:55,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:03:55,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:03:59,025 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:00,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:04:04,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:08,220 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:04:09,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:10,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:04:10,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:04:10,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:04:15,026 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:20,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:04:21,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:25,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:04:25,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:04:26,851 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:30,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:04:32,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:38,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:38,221 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:04:40,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:04:40,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:04:40,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:04:44,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:50,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:50,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:04:55,027 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:04:55,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:04:55,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:05:00,028 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:00,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:05:06,028 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:08,226 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:05:10,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:05:10,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:05:10,681 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:05:11,865 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:17,028 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:20,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:05:22,029 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:25,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:05:25,682 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:05:27,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:30,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:05:33,029 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:38,029 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:38,233 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:05:40,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:05:40,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:05:40,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:05:43,031 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:49,031 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:50,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:05:55,030 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:05:55,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:05:55,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:06:00,030 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:00,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:06:05,039 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:08,238 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:06:10,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:10,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:06:10,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:06:10,682 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:06:16,035 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:20,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:06:22,035 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:25,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:06:25,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:06:27,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:30,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:06:33,035 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:38,037 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:38,243 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:06:40,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:06:40,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:06:40,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:06:44,036 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:49,037 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:50,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:06:53,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:06:53,411 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:06:53,413 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:06:53,419 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:06:54,429 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:06:54,432 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:06:54,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:06:55,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:06:55,684 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:07:00,036 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:00,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:07:05,037 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:08,249 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:07:10,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:10,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:07:10,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:07:10,684 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:07:16,037 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:20,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:07:22,037 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:25,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:07:25,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:07:27,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:30,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:07:33,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:38,255 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:07:38,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:40,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:07:40,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:07:40,684 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:07:44,037 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:50,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:50,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:07:55,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:07:55,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:07:55,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:08:00,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:08:01,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:07,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:08,261 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:08:10,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:08:10,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:08:10,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:08:12,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:18,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:20,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:08:23,039 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:25,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:08:25,685 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:08:29,039 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:30,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:08:35,039 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:38,263 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:08:40,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:40,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:08:40,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:08:40,684 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:08:46,040 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:50,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:08:51,040 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:08:55,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:08:55,683 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:08:56,848 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:00,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:09:02,041 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:08,041 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:08,274 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:09:10,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:09:10,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:09:10,684 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:09:13,041 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:19,041 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:20,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:09:25,042 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:25,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:09:25,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:09:30,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:09:31,042 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:37,042 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:38,280 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:09:40,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:09:40,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:09:40,685 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:09:42,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:48,042 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:50,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:09:54,042 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:09:55,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:09:55,684 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:09:59,042 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:00,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:10:05,043 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:08,286 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:10:10,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:10,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:10:10,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:10:10,684 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:10:16,043 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:20,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:10:22,043 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:25,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:10:25,685 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:10:27,922 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:30,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:10:33,043 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:38,292 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:10:38,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:40,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:10:40,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:10:40,685 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:10:44,043 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:49,044 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:50,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:10:55,044 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:10:55,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:10:55,686 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:11:00,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:11:01,044 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:06,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:08,299 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:11:10,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:11:10,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:11:10,686 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:11:11,874 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:17,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:20,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:11:22,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:25,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:11:25,686 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:11:27,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:30,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:11:33,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:38,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:38,303 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:11:40,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:11:40,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:11:40,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:11:44,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:49,046 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:50,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:11:55,046 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:11:55,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:11:55,686 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:12:00,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:12:01,045 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:06,046 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:08,308 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:12:10,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:12:10,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:12:10,686 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:12:11,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:17,048 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:20,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:12:23,047 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:25,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:12:25,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:12:26,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:12:26,302 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:12:26,305 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:12:26,308 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:12:27,176 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:12:27,177 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:12:29,047 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:30,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:12:35,048 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:38,317 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:12:40,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:40,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:12:40,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:12:40,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:12:46,048 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:50,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:12:52,048 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:12:55,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:12:55,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:12:57,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:00,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:13:03,048 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:08,048 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:08,321 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:13:10,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:13:10,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:13:10,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:13:13,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:19,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:20,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:13:25,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:25,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:13:25,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:13:30,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:13:31,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:37,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:38,327 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:13:40,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:13:40,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:13:40,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:13:42,843 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:48,049 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:50,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:13:54,050 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:13:55,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:13:55,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:14:00,050 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:00,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:14:06,050 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:08,333 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:14:10,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:14:10,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:14:10,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:14:11,883 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:17,051 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:20,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:14:23,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:25,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:14:25,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:14:29,051 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:30,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:14:35,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:38,340 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:14:40,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:14:40,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:40,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:14:40,688 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:14:46,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:50,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:14:51,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:14:55,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:14:55,687 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:14:56,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:00,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:15:02,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:08,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:08,341 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:15:10,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:15:10,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:15:10,688 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:15:13,052 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:18,053 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:20,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:15:24,053 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:25,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:15:25,688 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:15:30,053 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:30,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:15:36,054 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:38,343 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:15:40,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:15:40,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:15:40,688 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:15:41,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:47,054 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:50,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:15:53,053 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:15:55,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:15:55,688 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:15:59,054 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:00,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:16:04,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:08,350 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:16:09,353 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:10,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:16:10,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:16:10,690 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:16:15,054 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:20,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:16:21,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:25,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:16:25,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:16:26,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:30,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:16:32,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:38,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:38,351 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:16:40,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:16:40,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:16:40,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:16:44,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:50,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:16:50,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:16:55,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:16:55,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:16:55,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:00,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:17:01,056 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:07,056 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:08,354 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:17:10,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:17:10,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:17:10,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:17:12,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:18,056 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:20,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:17:23,056 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:25,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:17:25,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:17:29,056 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:30,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:17:35,056 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:38,357 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:17:40,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:17:40,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:40,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:17:40,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:17:46,057 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:50,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:17:51,057 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:55,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:17:55,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:17:56,864 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:17:57,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:17:57,627 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:17:57,628 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:17:57,633 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:17:57,796 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:17:58,799 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:18:00,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:18:02,058 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:08,057 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:08,361 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:18:10,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:18:10,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:18:10,689 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:18:13,058 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:19,058 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:20,328 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:18:25,058 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:25,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:18:25,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:18:30,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:18:31,058 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:37,058 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:38,367 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:18:40,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:18:40,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:18:40,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:18:42,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:48,059 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:50,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:18:54,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:18:55,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:18:55,690 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:19:00,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:00,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:19:05,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:08,369 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:19:10,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:19:10,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:10,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:19:10,690 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:19:16,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:20,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:19:21,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:25,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:19:25,690 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:19:26,852 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:30,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:19:32,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:38,060 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:38,371 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:19:40,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:19:40,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:19:40,690 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:19:44,061 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:50,061 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:19:50,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:19:55,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:19:55,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:19:55,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:00,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:20:01,061 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:06,062 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:08,372 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:20:10,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:20:10,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:20:10,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:20:11,936 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:17,062 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:20,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:20:23,062 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:25,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:20:25,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:20:28,062 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:30,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:20:34,062 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:38,375 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:20:39,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:40,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:20:40,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:20:40,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:20:45,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:50,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:20:51,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:20:55,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:20:55,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:20:56,854 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:00,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:21:02,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:08,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:08,380 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:21:10,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:21:10,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:21:10,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:21:13,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:19,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:20,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:21:25,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:25,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:21:25,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:21:30,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:21:31,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:37,063 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:38,386 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:21:40,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:21:40,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:21:40,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:21:42,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:48,064 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:50,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:21:54,064 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:21:55,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:21:55,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:22:00,064 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:00,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:22:05,064 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:08,391 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:22:10,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:22:10,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:10,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:22:10,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:22:16,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:20,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:22:22,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:25,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:22:25,692 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:22:27,853 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:30,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:22:33,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:38,399 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:22:38,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:40,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:22:40,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:22:40,692 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:22:44,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:50,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:50,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:22:55,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:22:55,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:22:55,692 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:23:00,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:23:01,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:06,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:08,402 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:23:10,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:23:10,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:23:10,692 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:23:11,861 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:17,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:20,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:23:22,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:25,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:23:25,692 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:23:27,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:30,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:23:30,120 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:23:30,120 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:23:30,124 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:23:30,144 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:23:30,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:23:31,146 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:23:33,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:38,408 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:23:38,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:40,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:23:40,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:23:40,692 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:23:44,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:50,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:23:50,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:23:55,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:23:55,692 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:23:55,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:00,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:24:01,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:07,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:08,414 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:24:10,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:24:10,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:24:10,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:24:12,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:18,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:20,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:24:24,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:25,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:24:25,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:24:29,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:30,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:24:35,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:38,420 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:24:40,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:24:40,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:40,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:24:40,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:24:46,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:50,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:24:52,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:24:55,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:24:55,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:24:57,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:00,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:25:03,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:08,426 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:25:08,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:10,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:25:10,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:25:10,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:25:14,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:20,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:20,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:25:25,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:25:25,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:25:25,857 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:30,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:25:31,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:36,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:38,431 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:25:40,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:25:40,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:25:40,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:25:41,892 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:47,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:50,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:25:52,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:25:55,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:25:55,693 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:25:57,895 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:00,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:26:03,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:08,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:08,438 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:26:10,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:26:10,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:26:10,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:26:14,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:20,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:20,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:26:25,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:26:25,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:26:25,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:30,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:26:31,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:37,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:38,443 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:26:40,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:26:40,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:26:40,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:26:42,856 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:48,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:50,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:26:54,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:26:55,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:26:55,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:27:00,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:00,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:27:05,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:08,449 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:27:10,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:27:10,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:10,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:27:10,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:27:16,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:20,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:27:22,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:25,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:27:25,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:27:27,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:30,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:27:33,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:38,449 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:27:38,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:40,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:27:40,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:27:40,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:27:44,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:50,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:27:50,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:27:55,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:27:55,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:27:55,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:00,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:28:01,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:06,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:08,455 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:28:10,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:28:10,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:28:10,694 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:28:11,849 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:17,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:20,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:28:23,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:25,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:28:25,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:28:28,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:30,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:28:34,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:38,461 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:28:39,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:40,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:28:40,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:28:40,695 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:28:45,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:50,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:28:51,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:28:55,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:28:55,695 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:28:56,871 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:00,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:29:02,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:02,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:29:02,421 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:29:02,422 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:29:02,425 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:29:03,416 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:29:03,417 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:29:08,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:08,467 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:29:10,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:29:10,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:29:10,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:29:14,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:19,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:20,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:29:25,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:25,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:29:25,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:29:30,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:29:31,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:36,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:38,473 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:29:40,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:29:40,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:29:40,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:29:41,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:47,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:50,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:29:53,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:29:55,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:29:55,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:29:59,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:00,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:30:05,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:08,478 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:30:10,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:30:10,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:10,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:30:10,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:30:16,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:20,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:30:22,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:25,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:30:25,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:30:27,885 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:30,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:30:33,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:38,484 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:30:38,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:40,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:30:40,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:30:40,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:30:44,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:49,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:50,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:30:55,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:30:55,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:30:55,696 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:31:00,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:31:01,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:07,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:08,490 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:31:10,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:31:10,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:31:10,697 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:31:12,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:18,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:20,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:31:23,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:25,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:31:25,697 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:31:28,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:30,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:31:33,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:38,495 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:31:38,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:40,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:31:40,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:31:40,697 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:31:44,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:49,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:50,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:31:55,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:31:55,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:31:55,697 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:32:00,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:00,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:32:06,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:08,501 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:32:10,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:32:10,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:32:10,698 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:32:11,882 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:17,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:20,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:32:23,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:25,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:32:25,698 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:32:29,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:30,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:32:34,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:38,509 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:32:39,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:40,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:32:40,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:32:40,697 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:32:45,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:50,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:32:51,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:32:55,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:32:55,698 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:32:56,870 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:00,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:33:02,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:08,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:08,513 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:33:10,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:33:10,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:33:10,698 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:33:14,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:20,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:20,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:33:25,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:33:25,698 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:33:25,893 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:30,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:33:31,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:37,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:38,518 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:33:40,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:33:40,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:33:40,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:33:42,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:48,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:50,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:33:54,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:33:55,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:33:55,698 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:34:00,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:00,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:34:06,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:08,521 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:34:10,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:34:10,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:34:10,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:34:11,870 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:17,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:20,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:34:22,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:25,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:34:25,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:34:27,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:30,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:34:33,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:33,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:34:33,539 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:34:33,540 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:34:33,544 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:34:33,678 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:34:35,683 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:34:38,526 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:34:38,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:40,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:34:40,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:34:40,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:34:44,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:50,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:34:50,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:34:55,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:34:55,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:34:55,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:00,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:35:01,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:07,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:08,532 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:35:10,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:35:10,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:35:10,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:35:12,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:18,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:20,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:35:24,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:25,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:35:25,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:35:30,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:30,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:35:36,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:38,538 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:35:40,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:35:40,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:35:40,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:35:41,876 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:47,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:50,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:35:52,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:35:55,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:35:55,699 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:35:57,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:00,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:36:03,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:08,543 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:36:08,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:10,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:36:10,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:36:10,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:36:14,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:19,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:20,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:36:25,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:25,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:36:25,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:36:30,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:36:31,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:37,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:38,549 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:36:40,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:36:40,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:36:40,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:36:42,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:48,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:50,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:36:53,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:36:55,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:36:55,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:36:59,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:00,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:37:05,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:08,555 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:37:10,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:37:10,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:10,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:37:10,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:37:16,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:20,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:37:22,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:25,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:37:25,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:37:27,955 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:30,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:37:33,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:38,562 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:37:38,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:40,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:37:40,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:37:40,700 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:37:44,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:50,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:37:50,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:37:55,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:37:55,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:37:55,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:00,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:38:01,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:07,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:08,567 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:38:10,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:38:10,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:38:10,709 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:38:12,967 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:18,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:20,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:38:24,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:25,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:38:25,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:38:30,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:30,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:38:36,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:38,573 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:38:40,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:38:40,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:38:40,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:38:41,852 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:47,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:50,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:38:53,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:38:55,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:38:55,708 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:38:59,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:00,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:39:04,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:08,578 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:39:09,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:10,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:39:10,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:39:10,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:39:15,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:20,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:20,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:39:25,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:39:25,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:39:25,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:30,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:39:31,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:37,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:38,582 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:39:40,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:39:40,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:39:40,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:39:42,956 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:48,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:50,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:39:54,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:39:55,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:39:55,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:40:00,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:00,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:40:05,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:05,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:40:05,909 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:40:05,910 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:40:05,913 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:40:06,100 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:40:08,116 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:40:08,589 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:40:10,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:40:10,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:10,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:40:10,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:40:16,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:20,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:40:22,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:25,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:40:25,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:40:27,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:30,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:40:33,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:38,595 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:40:38,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:40,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:40:40,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:40:40,711 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:40:44,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:49,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:50,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:40:55,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:40:55,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:40:55,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:41:00,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:41:01,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:07,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:08,600 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:41:10,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:41:10,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:41:10,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:41:12,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:18,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:20,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:41:24,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:25,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:41:25,729 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:41:30,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:30,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:41:35,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:38,602 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:41:40,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:41:40,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:40,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:41:40,709 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:41:46,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:50,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:41:52,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:41:55,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:41:55,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:41:57,896 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:00,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:42:03,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:08,611 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:42:08,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:10,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:42:10,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:42:10,717 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:42:14,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:20,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:20,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:42:25,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:42:25,709 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:42:25,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:30,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:42:31,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:37,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:38,617 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:42:40,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:42:40,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:42:40,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:42:42,873 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:48,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:50,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:42:53,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:42:55,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:42:55,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:42:59,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:00,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:43:05,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:08,618 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:43:10,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:43:10,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:10,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:43:10,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:43:16,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:20,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:43:22,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:25,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:43:25,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:43:27,869 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:30,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:43:33,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:38,627 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:43:38,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:40,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:43:40,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:43:40,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:43:44,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:49,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:50,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:43:55,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:43:55,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:43:55,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:44:00,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:44:01,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:07,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:08,630 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:44:10,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:44:10,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:44:10,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:44:12,898 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:18,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:20,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:44:24,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:25,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:44:25,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:44:30,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:30,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:44:36,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:38,632 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:44:40,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:44:40,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:44:40,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:44:41,869 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:47,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:50,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:44:53,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:44:55,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:44:55,707 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:44:59,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:00,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:45:04,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:08,642 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:45:09,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:10,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:45:10,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:45:10,708 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:45:15,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:20,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:20,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:45:25,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:45:25,706 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:45:25,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:30,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:45:31,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:36,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:36,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:45:36,948 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:45:36,949 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:45:36,953 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:45:37,667 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:45:38,641 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:45:38,674 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:45:41,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:44,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:45:44,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:45:44,186 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:45:46,725 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:45:47,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:53,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:54,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:45:55,769 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:45:58,870 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:45:59,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:45:59,185 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:46:04,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:46:04,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:08,648 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:46:10,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:14,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:46:14,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:46:14,188 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:46:15,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:21,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:24,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:46:27,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:29,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:46:29,186 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:46:32,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:34,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:46:38,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:38,654 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:46:44,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:44,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:46:44,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:46:44,187 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:46:50,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:54,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:46:56,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:46:59,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:46:59,186 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:47:01,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:04,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:47:07,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:08,659 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:47:13,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:14,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:47:14,187 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:47:14,230 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:47:18,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:24,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:24,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:47:29,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:47:29,188 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:47:29,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:34,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:47:35,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:38,664 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:47:40,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:44,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:47:44,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:47:44,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:47:45,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:51,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:54,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:47:57,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:47:59,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:47:59,189 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:48:02,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:04,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:48:08,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:08,669 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:48:14,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:48:14,187 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:48:14,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:48:14,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:20,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:24,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:48:26,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:29,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:48:29,187 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:48:31,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:34,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:48:37,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:38,675 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:48:43,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:44,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:48:44,188 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:48:44,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:48:48,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:54,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:48:54,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:48:59,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:48:59,188 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:48:59,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:04,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:49:05,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:08,681 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:49:11,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:14,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:49:14,188 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:49:14,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:49:16,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:22,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:24,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:49:27,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:29,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:49:29,188 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:49:32,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:34,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:49:38,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:38,687 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:49:44,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:49:44,189 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:49:44,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:49:44,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:50,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:54,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:49:56,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:49:59,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:49:59,189 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:50:01,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:04,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:50:07,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:08,693 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:50:12,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:14,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:50:14,189 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:50:14,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:50:17,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:23,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:24,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:50:29,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:50:29,189 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:50:29,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:34,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:50:35,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:38,699 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:50:40,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:44,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:50:44,189 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:50:44,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:50:45,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:51,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:54,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:50:56,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:50:59,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:50:59,189 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:51:01,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:04,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:51:07,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:08,705 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:51:13,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:14,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:51:14,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:51:14,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:51:18,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:24,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:24,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:51:26,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:51:26,410 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:51:26,411 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:51:26,415 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:51:26,547 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:51:28,555 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:51:29,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:51:29,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:51:29,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:34,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:51:35,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:38,711 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:51:41,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:44,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:51:44,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:51:44,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:51:46,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:52,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:54,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:51:58,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:51:59,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:51:59,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:52:03,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:04,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:52:08,718 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:52:08,724 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:14,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:52:14,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:52:14,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:52:14,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:20,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:24,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:52:25,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:29,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:52:29,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:52:30,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:34,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:52:36,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:38,723 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:52:42,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:44,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:52:44,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:52:44,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:52:47,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:53,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:52:54,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:52:59,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:52:59,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:52:59,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:04,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:53:05,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:08,729 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:53:11,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:14,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:53:14,192 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:53:14,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:53:16,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:22,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:24,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:53:28,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:29,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:53:29,190 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:53:33,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:34,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:53:38,734 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:53:38,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:44,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:53:44,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:53:44,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:53:44,353 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:50,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:54,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:53:55,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:53:59,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:53:59,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:54:00,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:04,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:54:06,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:08,737 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:54:12,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:14,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:54:14,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:54:14,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:54:17,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:23,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:24,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:54:28,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:29,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:54:29,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:54:33,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:34,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:54:38,743 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:54:38,746 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:44,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:54:44,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:54:44,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:54:44,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:50,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:54,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:54:55,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:54:59,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:54:59,192 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:55:00,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:04,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:55:06,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:08,749 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:55:12,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:14,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:55:14,192 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:55:14,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:55:17,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:23,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:24,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:55:28,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:29,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:55:29,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:55:33,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:34,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:55:38,755 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:55:38,760 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:44,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:55:44,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:55:44,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:55:44,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:50,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:54,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:55:56,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:55:59,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:55:59,191 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:56:01,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:04,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:56:07,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:08,760 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:56:13,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:14,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:56:14,192 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:56:14,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:56:18,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:24,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:24,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:56:29,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:56:29,192 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:56:29,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:34,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:56:35,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:38,768 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:56:41,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:44,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:56:44,193 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:56:44,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:56:46,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:52,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:54,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:56:57,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:56:58,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 21:56:58,325 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 21:56:58,326 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 21:56:58,329 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 21:56:59,064 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 21:56:59,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:56:59,192 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:57:01,069 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 21:57:02,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:04,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:57:08,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:08,772 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:57:14,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:57:14,193 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:57:14,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:57:14,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:20,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:24,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:57:26,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:29,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:57:29,193 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:57:31,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:34,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:57:37,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:38,778 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:57:43,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:44,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:57:44,193 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:57:44,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:57:48,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:54,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:57:54,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:57:59,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:57:59,193 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:57:59,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:04,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:58:05,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:08,784 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:58:11,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:14,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:58:14,193 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:58:14,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:58:16,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:22,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:24,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:58:28,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:29,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:58:29,193 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:58:33,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:34,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:58:38,789 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:58:38,794 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:44,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:58:44,194 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:58:44,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:58:44,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:50,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:54,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:58:56,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:58:59,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:58:59,194 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:59:01,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:04,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:59:07,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:08,795 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:59:13,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:14,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:59:14,194 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:59:14,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:59:18,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:24,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:24,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:59:29,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:59:29,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:59:29,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:34,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:59:35,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:38,801 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 21:59:41,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:44,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:59:44,194 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 21:59:44,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:59:46,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:52,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:54,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 21:59:58,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 21:59:59,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 21:59:59,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:00:03,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:04,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:00:08,806 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:00:08,812 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:14,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:00:14,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:00:14,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:00:14,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:20,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:24,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:00:26,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:29,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:00:29,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:00:31,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:34,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:00:37,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:38,812 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:00:43,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:44,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:00:44,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:00:44,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:00:48,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:54,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:00:54,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:00:59,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:00:59,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:00:59,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:04,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:01:05,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:08,818 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:01:11,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:14,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:01:14,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:01:14,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:01:16,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:22,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:24,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:01:28,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:29,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:01:29,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:01:33,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:34,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:01:38,825 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:01:38,828 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:44,194 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:01:44,195 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:01:44,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:01:44,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:50,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:54,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:01:56,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:01:59,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:01:59,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:02:01,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:04,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:02:07,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:08,827 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:02:13,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:14,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:02:14,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:02:14,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:02:18,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:24,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:24,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:02:29,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:02:29,197 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:02:29,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:29,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:02:29,456 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:02:29,457 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:02:29,460 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:02:30,311 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:02:30,313 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:02:34,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:02:35,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:38,831 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:02:41,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:44,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:02:44,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:02:44,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:02:46,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:52,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:54,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:02:57,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:02:59,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:02:59,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:03:02,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:04,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:03:08,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:08,837 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:03:13,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:14,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:03:14,197 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:03:14,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:03:18,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:24,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:24,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:03:29,195 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:03:29,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:03:29,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:34,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:03:35,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:38,843 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:03:41,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:44,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:03:44,197 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:03:44,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:03:46,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:52,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:54,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:03:58,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:03:59,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:03:59,197 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:04:03,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:04,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:04:08,849 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:04:08,854 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:14,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:04:14,197 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:04:14,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:04:14,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:20,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:24,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:04:25,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:29,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:04:29,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:04:30,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:34,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:04:36,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:38,855 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:04:41,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:44,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:04:44,197 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:04:44,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:04:46,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:52,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:54,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:04:57,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:04:59,196 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:04:59,198 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:05:02,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:04,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:05:08,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:08,859 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:05:14,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:05:14,197 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:05:14,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:05:14,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:20,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:24,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:05:26,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:29,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:05:29,198 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:05:31,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:34,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:05:37,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:38,863 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:05:43,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:44,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:05:44,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:05:44,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:05:48,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:54,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:05:54,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:05:59,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:05:59,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:06:00,769 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:04,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:06:06,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:08,871 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:06:12,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:14,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:06:14,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:06:14,220 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:06:17,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:23,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:24,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:06:29,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:06:29,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:06:29,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:34,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:06:35,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:38,876 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:06:41,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:44,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:06:44,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:06:44,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:06:46,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:52,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:54,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:06:58,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:06:59,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:06:59,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:07:03,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:04,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:07:08,882 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:07:08,887 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:14,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:07:14,198 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:07:14,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:07:14,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:20,222 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:24,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:07:25,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:29,198 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:07:29,199 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:07:30,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:34,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:07:36,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:38,887 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:07:42,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:44,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:07:44,200 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:07:44,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:07:47,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:53,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:07:54,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:07:59,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:07:59,200 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:07:59,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:01,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:08:01,598 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:08:01,599 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:08:01,609 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:08:01,689 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:08:02,692 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:08:04,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:08:05,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:08,893 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:08:11,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:14,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:08:14,200 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:08:14,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:08:16,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:22,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:24,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:08:28,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:29,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:08:29,200 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:08:33,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:34,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:08:38,898 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:08:38,903 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:44,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:08:44,200 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:08:44,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:08:44,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:50,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:54,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:08:56,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:08:59,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:08:59,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:09:01,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:04,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:09:07,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:08,905 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:09:13,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:14,199 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:09:14,200 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:09:14,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:09:18,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:24,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:24,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:09:29,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:09:29,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:09:29,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:34,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:09:35,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:38,910 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:09:41,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:44,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:09:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:09:44,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:09:46,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:52,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:54,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:09:57,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:09:59,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:09:59,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:10:02,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:04,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:10:08,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:08,916 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:10:14,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:10:14,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:10:14,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:10:14,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:20,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:24,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:10:26,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:29,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:10:29,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:10:31,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:34,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:10:37,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:38,922 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:10:43,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:44,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:10:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:10:44,227 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:10:48,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:54,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:10:54,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:10:59,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:10:59,209 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:10:59,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:04,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:11:05,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:08,928 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:11:11,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:14,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:11:14,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:11:14,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:11:16,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:22,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:24,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:11:28,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:29,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:11:29,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:11:33,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:34,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:11:38,935 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:11:38,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:44,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:11:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:11:44,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:11:44,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:50,230 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:54,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:11:55,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:11:59,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:11:59,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:12:00,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:04,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:12:06,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:08,939 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:12:12,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:14,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:12:14,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:12:14,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:12:17,353 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:23,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:24,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:12:29,201 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:12:29,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:12:29,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:34,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:12:35,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:38,945 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:12:41,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:44,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:12:44,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:12:44,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:12:46,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:52,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:54,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:12:58,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:12:59,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:12:59,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:13:03,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:04,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:13:08,947 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:13:08,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:14,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:13:14,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:13:14,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:13:14,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:20,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:24,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:13:26,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:29,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:13:29,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:13:31,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:33,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:13:33,478 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:13:33,479 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:13:33,483 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:13:33,946 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:13:34,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:13:34,949 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:13:37,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:38,953 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:13:42,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:44,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:13:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:13:44,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:13:47,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:53,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:13:54,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:13:59,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:13:59,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:13:59,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:04,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:14:05,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:08,959 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:14:10,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:14,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:14:14,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:14:14,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:14:15,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:21,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:24,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:14:26,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:29,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:14:29,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:14:31,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:34,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:14:37,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:38,964 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:14:43,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:44,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:14:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:14:44,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:14:48,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:54,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:14:54,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:14:59,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:14:59,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:14:59,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:04,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:15:05,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:08,970 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:15:11,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:14,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:15:14,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:15:14,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:15:16,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:22,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:24,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:15:28,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:29,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:15:29,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:15:33,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:34,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:15:38,975 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:15:38,980 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:44,203 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:15:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:15:44,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:15:44,359 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:50,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:54,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:15:55,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:15:59,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:15:59,209 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:16:00,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:04,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:16:06,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:08,982 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:16:12,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:14,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:16:14,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:16:14,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:16:17,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:23,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:24,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:16:29,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:16:29,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:16:29,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:34,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:16:35,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:38,987 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:16:41,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:44,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:16:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:16:44,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:16:46,396 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:52,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:54,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:16:58,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:16:59,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:16:59,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:17:03,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:04,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:17:08,993 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:17:08,997 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:14,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:17:14,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:17:14,238 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:17:14,386 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:20,239 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:24,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:17:26,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:29,204 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:17:29,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:17:31,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:34,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:17:37,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:38,998 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:17:43,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:44,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:17:44,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:17:44,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:17:48,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:54,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:17:54,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:17:59,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:17:59,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:17:59,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:04,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:18:05,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:09,004 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:18:10,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:14,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:18:14,206 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:18:14,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:18:15,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:21,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:24,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:18:26,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:29,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:18:29,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:18:31,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:34,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:18:37,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:39,009 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:18:43,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:44,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:18:44,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:18:44,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:18:48,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:54,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:18:54,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:18:59,205 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:18:59,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:18:59,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:04,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:19:04,764 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:19:04,765 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:19:04,766 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:19:04,766 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:04,770 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:19:05,345 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:19:07,354 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:19:09,016 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:19:10,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:14,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:19:14,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:19:14,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:19:15,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:21,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:24,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:19:27,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:29,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:19:29,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:19:32,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:34,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:19:38,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:39,021 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:19:43,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:44,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:19:44,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:19:44,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:19:48,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:54,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:19:54,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:19:59,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:19:59,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:19:59,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:04,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:20:05,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:09,027 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:20:11,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:14,206 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:20:14,207 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:20:14,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:20:16,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:22,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:24,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:20:28,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:29,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:20:29,209 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:20:33,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:34,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:20:39,031 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:20:39,036 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:44,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:20:44,212 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:20:44,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:20:44,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:50,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:54,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:20:55,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:20:59,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:20:59,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:21:00,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:04,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:21:06,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:09,039 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:21:12,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:14,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:21:14,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:21:14,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:21:17,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:23,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:24,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:21:28,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:29,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:21:29,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:21:33,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:34,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:21:39,043 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:21:39,047 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:44,207 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:21:44,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:21:44,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:21:44,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:50,247 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:54,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:21:55,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:21:59,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:21:59,209 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:22:00,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:04,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:22:06,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:09,047 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:22:12,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:14,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:22:14,208 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:22:14,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:22:17,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:23,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:24,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:22:29,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:22:29,209 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:22:29,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:34,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:22:35,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:39,052 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:22:41,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:44,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:22:44,210 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:22:44,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:22:46,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:52,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:54,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:22:57,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:22:59,208 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:22:59,209 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:23:02,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:04,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:23:08,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:09,059 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:23:14,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:23:14,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:23:14,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:23:14,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:20,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:24,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:23:25,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:29,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:23:29,210 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:23:30,363 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:34,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:23:36,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:39,063 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:23:42,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:44,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:23:44,213 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:23:44,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:23:47,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:53,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:54,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:23:58,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:23:59,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:23:59,210 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:24:03,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:04,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:24:09,069 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:24:09,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:14,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:24:14,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:24:14,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:24:14,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:20,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:24,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:24:26,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:29,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:24:29,210 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:24:31,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:34,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:24:37,024 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:24:37,029 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:24:37,030 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:24:37,030 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:37,034 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:24:37,552 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:24:39,074 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:24:39,565 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:24:42,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:44,209 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:24:44,210 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:24:44,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:24:47,353 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:53,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:54,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:24:58,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:24:59,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:24:59,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:25:03,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:04,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:25:09,080 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:25:09,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:14,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:25:14,212 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:25:14,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:25:14,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:20,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:24,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:25:26,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:29,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:25:29,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:25:31,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:34,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:25:37,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:39,086 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:25:42,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:44,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:25:44,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:25:44,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:25:47,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:53,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:25:54,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:25:59,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:25:59,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:25:59,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:04,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:26:05,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:09,097 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:26:10,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:14,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:26:14,212 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:26:14,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:26:15,790 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:21,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:24,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:26:27,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:29,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:26:29,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:26:32,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:34,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:26:38,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:39,097 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:26:44,210 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:26:44,211 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:26:44,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:26:44,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:50,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:54,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:26:56,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:26:59,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:26:59,212 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:27:01,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:04,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:27:07,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:09,103 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:27:13,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:14,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:27:14,212 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:27:14,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:27:18,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:24,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:24,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:27:29,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:27:29,212 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:27:29,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:34,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:27:35,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:39,108 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:27:41,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:44,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:27:44,213 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:27:44,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:27:46,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:52,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:54,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:27:58,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:27:59,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:27:59,213 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:28:03,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:04,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:28:09,114 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:28:09,119 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:14,211 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:28:14,213 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:28:14,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:28:14,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:20,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:24,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:28:26,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:29,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:28:29,213 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:28:31,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:34,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:28:37,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:39,120 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:28:43,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:44,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:28:44,213 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:28:44,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:28:48,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:54,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:28:54,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:28:59,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:28:59,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:28:59,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:04,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:29:05,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:09,126 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:29:10,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:14,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:29:14,213 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:29:14,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:29:15,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:21,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:24,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:29:27,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:29,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:29:29,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:29:32,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:34,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:29:38,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:39,130 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:29:44,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:29:44,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:29:44,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:29:44,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:50,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:54,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:29:55,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:29:59,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:29:59,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:30:00,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:04,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:30:06,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:08,011 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:30:08,016 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:30:08,017 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:30:08,021 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:30:08,760 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:30:09,136 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:30:09,762 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:30:12,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:14,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:30:14,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:30:14,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:30:17,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:23,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:24,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:30:28,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:29,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:30:29,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:30:33,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:34,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:30:39,142 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:30:39,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:44,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:30:44,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:30:44,266 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:30:44,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:50,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:54,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:30:55,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:30:59,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:30:59,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:31:00,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:04,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:31:06,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:09,150 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:31:12,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:14,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:31:14,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:31:14,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:31:17,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:23,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:24,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:31:29,213 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:31:29,214 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:31:29,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:34,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:31:35,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:39,154 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:31:41,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:44,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:31:44,215 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:31:44,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:31:46,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:52,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:54,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:31:57,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:31:59,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:31:59,215 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:32:02,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:04,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:32:08,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:09,159 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:32:14,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:32:14,216 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:32:14,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:32:14,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:20,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:24,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:32:25,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:29,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:32:29,215 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:32:30,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:34,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:32:36,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:39,165 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:32:42,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:44,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:32:44,217 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:32:44,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:32:47,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:53,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:32:54,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:32:59,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:32:59,216 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:32:59,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:04,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:33:05,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:09,171 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:33:11,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:14,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:33:14,216 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:33:14,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:33:16,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:22,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:24,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:33:28,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:29,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:33:29,216 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:33:33,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:34,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:33:39,177 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:33:39,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:44,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:33:44,216 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:33:44,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:33:44,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:50,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:54,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:33:56,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:33:59,215 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:33:59,216 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:34:01,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:04,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:34:07,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:09,183 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:34:13,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:14,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:34:14,216 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:34:14,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:34:18,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:24,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:24,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:34:29,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:34:29,217 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:34:29,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:34,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:34:35,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:39,188 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:34:41,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:44,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:34:44,217 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:34:44,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:34:46,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:52,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:54,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:34:58,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:34:59,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:34:59,217 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:35:03,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:04,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:35:09,194 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:35:09,200 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:14,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:35:14,218 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:35:14,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:35:14,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:20,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:24,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:35:26,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:29,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:35:29,218 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:35:31,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:34,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:35:37,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:39,201 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:35:40,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:35:40,265 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:35:40,266 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:35:40,269 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:35:40,977 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:35:41,980 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:35:43,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:44,216 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:35:44,217 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:35:44,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:35:48,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:54,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:35:54,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:35:59,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:35:59,218 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:35:59,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:04,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:36:05,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:09,205 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:36:11,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:14,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:36:14,218 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:36:14,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:36:16,390 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:22,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:24,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:36:27,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:29,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:36:29,218 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:36:32,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:34,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:36:38,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:39,211 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:36:44,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:36:44,218 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:36:44,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:36:44,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:50,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:54,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:36:55,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:36:59,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:36:59,219 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:37:00,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:04,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:37:06,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:09,218 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:37:12,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:14,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:37:14,219 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:37:14,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:37:17,369 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:23,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:24,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:37:29,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:37:29,218 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:37:29,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:34,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:37:35,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:39,224 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:37:41,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:44,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:37:44,219 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:37:44,278 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:37:46,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:52,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:54,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:37:58,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:37:59,218 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:37:59,219 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:38:03,397 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:04,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:38:09,230 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:38:09,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:14,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:38:14,221 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:38:14,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:38:14,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:20,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:24,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:38:26,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:29,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:38:29,220 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:38:31,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:34,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:38:37,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:39,237 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:38:43,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:44,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:38:44,220 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:38:44,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:38:48,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:54,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:38:54,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:38:59,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:38:59,220 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:38:59,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:04,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:39:05,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:09,242 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:39:10,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:14,219 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:39:14,220 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:39:14,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:39:15,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:21,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:24,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:39:27,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:30,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:39:30,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:39:32,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:34,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:39:38,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:39,247 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:39:44,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:44,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:39:45,064 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:39:45,074 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:39:49,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:39:54,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:39:55,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:00,064 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:40:00,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:40:00,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:04,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:40:06,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:09,254 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:40:12,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:14,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:40:15,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:40:15,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:40:17,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:23,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:24,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:40:29,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:30,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:40:30,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:40:34,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:34,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:40:39,268 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:40:40,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:44,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:40:45,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:40:45,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:40:45,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:51,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:40:54,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:40:56,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:00,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:41:00,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:41:01,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:04,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:41:07,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:09,266 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:41:12,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:41:12,498 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:41:12,499 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:41:12,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:12,508 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:41:13,289 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:41:14,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:41:14,292 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:41:15,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:41:15,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:41:18,212 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:24,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:24,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:41:30,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:41:30,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:41:30,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:34,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:41:36,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:39,270 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:41:42,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:44,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:41:45,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:41:45,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:41:47,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:53,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:41:54,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:41:59,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:00,065 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:42:00,066 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:42:04,236 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:04,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:42:09,280 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:42:09,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:14,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:42:15,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:42:15,067 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:42:15,230 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:21,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:24,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:42:27,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:30,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:42:30,067 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:42:32,253 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:34,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:42:38,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:39,283 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:42:44,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:44,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:42:45,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:42:45,067 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:42:49,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:42:54,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:42:55,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:00,066 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:43:00,067 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:43:00,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:04,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:43:06,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:09,289 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:43:12,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:14,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:43:15,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:43:15,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:43:17,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:23,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:24,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:43:29,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:30,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:43:30,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:43:34,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:34,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:43:39,295 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:43:39,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:44,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:43:45,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:43:45,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:43:45,229 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:51,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:43:54,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:43:57,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:00,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:44:00,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:44:02,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:04,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:44:08,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:09,301 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:44:14,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:14,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:44:15,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:44:15,069 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:44:19,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:24,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:44:25,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:30,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:44:30,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:44:30,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:34,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:44:36,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:39,307 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:44:41,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:44,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:44:45,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:44:45,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:44:46,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:52,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:44:54,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:44:57,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:00,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:45:00,069 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:45:02,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:04,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:45:08,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:09,313 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:45:14,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:14,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:45:15,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:45:15,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:45:19,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:24,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:45:25,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:30,067 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:45:30,068 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:45:30,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:34,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:45:36,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:39,320 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:45:41,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:44,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:45:45,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:45:45,069 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:45:46,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:52,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:45:54,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:45:58,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:00,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:46:00,069 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:46:03,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:04,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:46:09,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:09,321 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:46:14,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:46:15,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:46:15,069 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:46:15,231 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:21,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:24,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:46:27,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:30,068 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:46:30,069 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:46:32,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:34,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:46:38,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:39,324 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:46:43,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:43,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:46:43,443 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:46:43,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:46:43,448 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:46:43,545 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:46:44,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:46:44,548 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:46:45,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:46:45,070 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:46:48,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:54,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:46:54,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:46:59,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:00,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:47:00,070 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:47:04,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:04,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:47:09,326 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:47:09,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:14,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:47:15,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:47:15,070 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:47:15,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:21,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:24,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:47:26,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:30,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:47:30,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:47:31,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:34,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:47:37,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:39,331 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:47:43,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:44,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:47:45,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:47:45,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:47:48,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:54,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:47:54,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:48:00,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:48:00,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:48:00,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:04,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:48:06,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:09,337 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:48:12,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:14,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:48:15,069 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:48:15,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:48:17,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:23,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:24,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:48:29,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:30,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:48:30,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:48:34,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:34,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:48:39,343 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:48:39,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:44,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:48:45,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:48:45,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:48:45,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:51,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:48:54,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:48:57,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:00,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:49:00,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:49:02,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:04,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:49:08,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:09,349 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:49:13,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:14,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:49:15,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:49:15,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:49:18,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:24,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:24,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:49:30,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:49:30,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:49:30,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:34,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:49:36,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:39,354 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:49:42,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:44,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:49:45,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:49:45,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:49:47,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:53,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:49:54,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:49:59,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:00,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:50:00,072 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:50:04,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:04,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:50:09,358 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:50:09,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:14,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:50:15,070 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:50:15,071 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:50:15,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:21,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:24,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:50:27,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:30,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:50:30,072 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:50:32,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:34,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:50:38,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:39,367 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:50:44,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:44,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:50:45,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:50:45,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:50:49,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:50:54,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:50:55,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:00,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:51:00,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:51:00,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:04,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:51:06,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:09,371 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:51:12,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:14,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:51:15,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:51:15,074 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:51:17,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:23,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:24,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:51:28,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:30,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:51:30,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:51:33,234 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:34,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:51:39,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:39,377 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:51:44,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:51:45,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:51:45,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:51:45,235 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:51,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:51:54,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:51:57,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:00,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:52:00,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:52:02,221 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:04,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:52:08,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:09,383 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:52:14,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:14,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:52:15,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:52:15,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:52:15,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:52:15,365 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:52:15,366 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:52:15,369 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:52:15,866 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:52:16,882 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:52:19,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:24,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:52:25,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:30,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:52:30,074 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:52:30,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:34,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:52:36,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:39,389 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:52:42,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:44,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:52:45,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:52:45,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:52:47,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:53,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:52:54,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:52:59,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:00,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:53:00,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:53:04,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:53:04,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:09,394 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:53:09,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:14,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:53:15,072 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:53:15,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:53:15,232 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:21,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:24,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:53:27,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:30,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:53:30,073 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:53:32,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:34,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:53:38,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:39,400 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:53:43,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:44,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:53:45,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:53:45,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:53:48,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:54,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:53:54,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:54:00,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:54:00,074 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:54:00,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:04,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:54:06,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:09,406 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:54:12,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:14,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:54:15,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:54:15,074 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:54:17,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:23,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:24,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:54:29,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:30,073 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:54:30,074 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:54:34,280 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:34,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:54:39,411 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:54:39,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:44,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:54:45,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:54:45,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:54:45,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:51,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:54:54,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:54:57,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:00,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:55:00,075 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:55:02,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:04,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:55:08,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:09,413 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:55:14,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:14,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:55:15,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:55:15,075 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:55:19,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:24,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:55:25,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:30,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:55:30,075 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:55:30,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:34,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:55:36,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:39,420 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:55:42,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:44,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:55:45,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:55:45,075 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:55:47,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:53,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:55:54,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:55:59,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:00,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:56:00,075 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:56:04,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:04,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:56:09,420 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:56:09,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:14,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:56:15,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:56:15,075 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:56:15,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:21,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:24,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:56:27,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:30,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:56:30,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:56:32,224 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:34,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:56:38,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:39,422 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:56:44,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:44,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:56:45,074 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:56:45,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:56:49,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:56:54,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:56:55,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:00,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:57:00,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:57:00,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:04,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:57:06,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:09,424 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:57:12,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:14,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:57:15,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:57:15,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:57:17,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:23,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:24,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:57:28,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:30,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:57:30,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:57:33,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:34,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:57:39,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:39,426 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:57:44,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:44,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:57:45,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:57:45,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:57:47,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 22:57:47,376 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 22:57:47,376 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 22:57:47,380 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 22:57:48,229 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 22:57:49,232 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 22:57:49,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:57:54,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:57:55,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:00,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:58:00,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:58:00,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:04,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:58:06,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:09,428 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:58:12,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:14,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:58:15,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:58:15,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:58:17,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:23,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:24,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:58:29,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:30,075 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:58:30,076 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:58:34,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:34,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:58:39,430 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:58:39,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:44,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:58:45,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:58:45,077 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:58:45,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:51,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:58:54,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:58:57,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:00,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:59:00,077 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:59:02,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:04,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:59:08,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:09,432 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:59:14,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:14,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:59:15,076 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:59:15,077 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:59:19,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:24,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:59:25,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:30,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:59:30,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:59:30,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:34,585 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:59:36,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:39,435 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 22:59:42,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:44,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:59:45,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 22:59:45,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 22:59:47,306 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:53,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 22:59:54,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 22:59:59,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:00,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:00:00,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:00:04,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:04,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:00:09,440 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:00:09,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:14,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:00:15,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:00:15,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:00:15,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:21,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:24,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:00:27,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:30,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:00:30,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:00:32,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:34,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:00:38,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:39,439 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:00:44,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:44,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:00:45,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:00:45,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:00:49,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:00:54,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:00:55,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:00,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:01:00,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:01:00,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:04,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:01:06,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:09,441 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:01:12,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:14,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:01:15,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:01:15,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:01:17,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:23,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:24,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:01:28,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:30,077 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:01:30,078 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:01:33,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:34,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:01:39,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:39,443 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:01:44,587 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:01:45,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:01:45,079 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:01:45,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:51,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:01:54,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:01:57,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:00,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:02:00,079 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:02:02,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:04,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:02:08,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:09,446 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:02:13,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:14,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:02:15,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:02:15,079 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:02:18,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:24,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:24,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:02:30,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:02:30,079 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:02:30,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:34,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:02:36,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:39,449 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:02:42,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:44,588 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:02:45,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:02:45,079 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:02:47,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:53,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:02:54,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:02:59,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:00,078 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:03:00,079 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:03:04,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:04,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:03:09,455 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:03:09,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:14,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:03:15,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:03:15,080 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:03:15,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:18,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:03:18,246 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:03:18,247 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:03:18,250 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:03:18,424 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:03:19,428 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:03:21,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:24,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:03:27,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:30,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:03:30,080 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:03:32,294 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:34,591 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:03:38,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:39,461 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:03:43,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:44,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:03:45,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:03:45,080 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:03:48,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:54,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:03:54,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:04:00,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:04:00,080 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:04:00,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:04,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:04:06,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:09,467 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:04:12,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:14,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:04:15,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:04:15,080 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:04:17,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:23,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:24,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:04:29,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:30,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:04:30,080 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:04:34,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:34,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:04:39,473 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:04:39,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:44,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:04:45,079 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:04:45,081 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:04:45,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:51,110 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:04:54,592 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:04:57,110 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:00,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:05:00,081 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:05:02,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:04,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:05:08,111 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:09,480 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:05:13,112 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:14,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:05:15,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:05:15,081 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:05:18,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:24,111 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:24,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:05:30,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:05:30,082 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:05:30,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:34,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:05:36,111 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:39,484 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:05:41,112 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:44,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:05:45,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:05:45,081 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:05:46,240 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:52,112 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:05:54,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:05:58,112 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:00,080 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:06:00,081 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:06:03,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:04,593 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:06:09,112 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:09,490 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:06:14,113 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:14,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:06:15,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:06:15,082 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:06:19,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:24,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:06:25,112 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:30,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:06:30,081 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:06:30,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:34,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:06:36,113 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:39,496 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:06:41,114 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:44,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:06:45,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:06:45,082 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:06:46,284 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:52,113 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:06:54,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:06:58,113 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:00,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:07:00,082 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:07:03,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:04,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:07:09,114 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:09,502 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:07:14,115 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:14,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:07:15,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:07:15,082 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:07:19,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:24,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:07:25,114 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:30,081 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:07:30,082 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:07:30,259 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:34,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:07:36,115 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:39,508 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:07:41,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:44,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:07:45,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:07:45,083 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:07:46,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:52,115 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:07:54,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:07:58,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:00,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:08:00,083 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:08:03,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:04,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:08:09,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:09,512 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:08:14,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:08:15,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:08:15,083 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:08:15,329 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:21,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:24,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:08:27,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:30,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:08:30,083 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:08:32,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:34,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:08:38,117 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:39,519 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:08:44,117 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:44,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:08:45,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:08:45,083 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:08:49,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:50,257 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:08:50,260 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:08:50,261 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:08:50,265 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:08:50,637 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:08:51,641 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:08:54,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:08:54,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:09:00,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:09:00,084 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:09:00,228 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:04,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:09:06,118 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:09,521 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:09:12,118 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:14,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:09:15,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:09:15,083 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:09:17,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:23,118 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:24,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:09:29,118 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:30,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:09:30,084 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:09:34,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:34,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:09:39,527 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:09:39,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:44,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:09:45,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:09:45,084 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:09:45,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:51,119 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:09:54,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:09:57,118 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:00,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:10:00,084 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:10:02,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:04,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:10:08,119 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:09,531 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:10:14,119 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:14,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:10:15,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:10:15,084 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:10:19,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:24,597 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:10:25,119 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:30,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:10:30,084 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:10:30,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:34,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:10:36,120 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:39,535 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:10:42,120 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:44,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:10:45,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:10:45,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:10:47,303 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:53,121 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:10:54,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:10:59,121 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:00,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:11:00,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:11:04,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:04,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:11:09,541 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:11:09,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:14,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:11:15,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:11:15,084 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:11:15,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:21,121 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:24,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:11:27,121 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:30,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:11:30,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:11:32,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:34,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:11:38,122 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:39,547 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:11:44,122 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:44,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:11:45,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:11:45,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:11:49,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:11:54,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:11:55,122 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:00,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:12:00,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:12:00,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:04,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:12:06,122 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:09,550 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:12:11,123 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:14,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:12:15,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:12:15,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:12:16,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:22,123 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:24,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:12:28,123 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:30,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:12:30,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:12:33,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:34,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:12:39,123 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:39,556 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:12:44,123 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:44,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:12:45,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:12:45,085 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:12:49,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:12:54,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:12:55,123 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:00,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:13:00,086 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:13:00,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:04,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:13:06,125 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:09,562 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:13:12,125 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:14,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:13:15,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:13:15,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:13:17,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:23,125 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:24,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:13:29,125 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:30,085 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:13:30,086 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:13:34,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:34,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:13:39,568 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:13:39,573 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:44,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:13:45,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:13:45,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:13:45,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:51,125 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:13:54,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:13:56,126 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:00,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:14:00,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:14:01,256 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:04,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:14:07,126 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:09,574 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:14:13,126 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:14,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:14:15,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:14:15,086 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:14:18,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:21,255 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:14:21,258 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:14:21,259 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:14:21,263 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:14:21,853 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:14:23,858 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:14:24,265 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:24,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:14:30,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:14:30,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:14:30,244 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:34,600 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:14:36,126 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:39,581 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:14:42,127 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:44,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:14:45,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:14:45,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:14:47,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:53,127 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:14:54,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:14:59,127 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:00,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:15:00,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:15:04,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:04,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:15:09,585 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:15:09,590 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:14,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:15:15,086 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:15:15,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:15:15,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:21,128 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:24,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:15:27,128 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:30,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:15:30,087 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:15:32,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:34,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:15:38,128 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:39,591 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:15:44,129 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:44,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:15:45,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:15:45,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:15:49,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:15:54,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:15:55,129 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:00,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:16:00,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:16:00,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:04,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:16:06,130 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:09,597 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:16:12,130 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:14,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:16:15,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:16:15,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:16:17,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:23,131 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:24,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:16:29,130 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:30,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:16:30,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:16:34,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:34,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:16:39,603 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:16:39,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:44,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:16:45,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:16:45,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:16:45,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:51,130 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:16:54,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:16:57,130 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:00,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:17:00,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:17:02,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:04,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:17:08,131 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:09,608 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:17:13,131 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:14,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:17:15,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:17:15,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:17:18,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:24,131 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:24,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:17:30,087 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:17:30,088 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:17:30,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:34,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:17:36,132 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:39,614 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:17:41,133 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:44,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:17:45,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:17:45,089 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:17:46,275 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:52,132 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:17:54,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:17:57,133 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:00,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:18:00,089 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:18:02,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:04,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:18:08,133 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:09,619 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:18:14,134 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:14,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:18:15,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:18:15,089 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:18:19,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:24,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:18:25,133 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:30,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:18:30,089 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:18:30,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:34,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:18:36,134 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:39,625 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:18:42,134 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:44,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:18:45,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:18:45,089 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:18:47,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:53,134 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:18:54,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:18:59,134 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:00,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:19:00,089 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:19:04,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:04,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:19:09,631 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:19:09,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:14,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:19:15,088 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:19:15,089 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:19:15,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:21,136 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:24,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:19:27,135 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:30,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:19:30,090 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:19:32,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:34,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:19:38,136 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:39,638 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:19:44,136 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:44,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:19:45,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:19:45,090 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:19:49,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:53,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:19:53,388 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:19:53,389 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:19:53,392 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:19:54,126 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:19:54,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:19:54,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:19:56,132 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:20:00,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:20:00,090 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:20:00,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:04,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:20:06,136 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:09,641 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:20:12,137 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:14,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:20:15,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:20:15,091 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:20:17,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:23,137 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:24,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:20:29,137 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:30,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:20:30,090 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:20:34,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:34,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:20:39,647 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:20:39,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:44,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:20:45,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:20:45,090 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:20:45,246 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:51,137 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:20:54,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:20:57,138 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:00,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:21:00,090 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:21:02,289 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:04,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:21:08,138 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:09,652 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:21:14,138 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:14,606 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:21:15,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:21:15,091 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:21:19,254 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:24,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:21:25,138 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:30,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:21:30,091 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:21:30,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:34,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:21:36,139 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:39,655 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:21:42,139 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:44,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:21:45,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:21:45,091 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:21:47,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:53,140 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:21:54,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:21:59,140 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:00,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:22:00,091 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:22:04,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:04,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:22:09,656 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:22:09,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:14,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:22:15,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:22:15,091 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:22:15,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:21,141 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:24,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:22:27,140 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:30,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:22:30,091 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:22:32,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:34,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:22:38,141 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:39,667 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:22:44,141 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:44,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:22:45,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:22:45,092 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:22:49,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:22:54,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:22:55,141 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:00,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:23:00,093 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:23:00,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:04,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:23:06,142 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:09,666 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:23:12,142 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:14,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:23:15,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:23:15,092 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:23:17,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:23,142 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:24,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:23:29,142 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:30,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:23:30,092 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:23:34,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:34,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:23:39,671 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:23:39,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:44,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:23:45,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:23:45,092 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:23:45,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:51,143 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:23:54,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:23:57,143 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:00,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:24:00,093 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:24:02,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:04,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:24:08,143 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:09,675 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:24:13,144 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:14,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:24:15,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:24:15,093 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:24:18,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:24,144 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:24,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:24:30,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:24:30,093 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:24:30,243 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:34,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:24:36,144 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:39,685 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:24:42,144 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:44,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:24:45,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:24:45,093 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:24:47,267 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:53,145 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:24:54,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:24:58,145 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:00,092 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:25:00,093 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:25:03,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:04,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:25:09,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:09,688 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:25:14,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:25:15,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:25:15,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:25:15,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:21,145 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:24,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:25:25,895 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:25:25,898 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:25:25,899 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:25:25,913 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:25:26,638 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:25:26,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:27,642 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:25:30,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:25:30,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:25:32,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:34,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:25:38,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:39,690 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:25:43,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:44,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:25:45,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:25:45,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:25:48,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:54,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:25:54,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:25:59,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:00,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:26:00,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:26:04,293 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:04,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:26:09,695 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:26:09,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:14,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:26:15,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:26:15,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:26:15,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:21,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:24,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:26:27,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:30,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:26:30,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:26:32,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:34,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:26:38,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:39,700 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:26:43,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:44,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:26:45,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:26:45,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:26:48,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:54,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:26:54,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:26:59,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:00,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:27:00,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:27:04,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:04,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:27:09,705 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:27:09,711 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:14,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:27:15,093 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:27:15,094 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:27:15,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:21,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:24,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:27:27,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:30,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:27:30,095 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:27:32,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:34,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:27:38,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:39,711 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:27:44,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:44,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:27:45,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:27:45,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:27:49,250 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:27:54,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:27:55,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:00,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:28:00,095 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:28:00,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:04,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:28:06,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:09,715 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:28:12,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:14,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:28:15,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:28:15,095 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:28:17,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:23,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:24,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:28:29,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:30,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:28:30,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:28:34,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:34,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:28:39,721 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:28:39,727 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:44,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:28:45,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:28:45,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:28:45,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:51,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:28:54,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:28:57,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:00,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:29:00,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:29:02,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:04,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:29:08,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:09,726 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:29:14,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:14,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:29:15,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:29:15,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:29:19,270 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:24,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:29:25,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:30,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:29:30,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:29:30,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:34,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:29:36,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:39,734 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:29:42,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:44,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:29:45,095 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:29:45,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:29:47,252 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:53,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:29:54,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:29:58,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:00,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:30:00,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:30:03,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:04,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:30:09,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:09,736 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:30:14,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:14,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:30:15,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:30:15,097 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:30:19,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:24,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:30:25,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:30,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:30:30,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:30:30,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:34,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:30:36,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:39,743 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:30:41,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:44,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:30:45,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:30:45,097 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:30:46,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:52,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:30:54,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:30:56,991 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:30:56,994 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:30:56,995 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:30:56,998 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:30:57,974 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:30:57,977 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:30:58,036 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:00,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:31:00,097 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:31:03,264 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:04,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:31:09,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:09,749 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:31:14,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:31:15,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:31:15,097 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:31:15,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:21,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:24,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:31:27,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:30,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:31:30,097 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:31:32,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:34,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:31:38,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:39,754 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:31:43,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:44,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:31:45,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:31:45,098 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:31:48,305 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:54,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:31:54,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:32:00,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:32:00,097 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:32:00,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:04,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:32:06,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:09,760 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:32:11,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:14,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:32:15,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:32:15,098 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:32:16,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:22,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:24,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:32:28,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:30,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:32:30,098 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:32:33,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:34,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:32:39,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:39,764 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:32:44,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:44,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:32:45,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:32:45,098 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:32:49,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:32:54,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:32:55,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:00,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:33:00,098 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:33:00,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:04,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:33:06,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:09,769 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:33:12,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:14,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:33:15,097 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:33:15,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:33:17,304 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:23,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:24,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:33:29,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:30,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:33:30,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:33:34,249 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:34,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:33:39,773 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:33:39,778 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:44,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:33:45,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:33:45,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:33:45,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:51,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:33:54,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:33:57,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:00,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:34:00,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:34:02,277 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:04,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:34:08,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:09,779 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:34:13,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:14,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:34:15,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:34:15,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:34:18,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:24,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:24,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:34:30,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:34:30,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:34:30,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:34,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:34:36,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:39,787 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:34:42,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:44,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:34:45,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:34:45,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:34:47,288 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:53,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:34:54,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:34:59,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:00,098 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:35:00,099 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:35:04,291 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:04,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:35:09,792 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:35:09,797 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:14,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:35:15,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:35:15,100 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:35:15,283 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:21,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:24,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:35:27,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:30,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:35:30,100 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:35:32,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:34,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:35:38,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:39,797 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:35:44,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:44,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:35:45,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:35:45,100 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:35:49,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:35:54,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:35:55,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:00,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:36:00,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:36:00,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:04,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:36:06,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:09,803 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:36:12,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:14,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:36:15,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:36:15,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:36:17,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:23,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:24,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:36:28,838 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:36:28,841 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:36:28,842 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:36:28,843 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:28,846 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:36:29,173 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:36:30,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:36:30,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:36:30,177 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:36:34,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:34,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:36:39,809 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:36:39,814 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:44,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:36:45,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:36:45,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:36:45,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:51,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:36:54,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:36:57,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:00,100 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:37:00,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:37:02,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:04,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:37:08,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:09,815 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:37:14,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:14,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:37:15,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:37:15,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:37:19,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:24,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:37:25,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:30,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:37:30,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:37:30,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:34,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:37:36,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:39,821 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:37:42,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:44,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:37:45,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:37:45,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:37:47,874 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:53,169 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:37:54,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:37:59,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:00,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:38:00,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:38:04,286 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:04,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:38:09,826 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:38:09,832 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:14,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:38:15,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:38:15,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:38:15,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:21,169 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:24,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:38:27,169 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:30,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:38:30,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:38:32,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:34,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:38:38,169 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:39,828 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:38:44,170 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:44,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:38:45,101 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:38:45,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:38:49,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:38:54,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:38:55,170 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:00,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:39:00,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:39:00,287 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:04,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:39:06,171 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:09,836 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:39:12,171 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:14,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:39:15,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:39:15,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:39:17,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:23,171 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:24,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:39:29,172 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:30,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:39:30,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:39:34,296 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:34,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:39:39,840 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:39:39,847 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:44,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:39:45,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:39:45,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:39:45,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:51,175 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:39:54,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:39:57,175 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:00,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:40:00,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:40:02,268 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:04,623 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:40:08,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:09,846 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:40:13,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:14,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:40:15,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:40:15,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:40:18,285 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:24,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:24,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:40:30,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:40:30,108 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:40:30,292 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:34,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:40:36,176 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:39,852 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:40:42,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:44,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:40:45,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:40:45,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:40:47,279 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:53,177 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:40:54,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:40:59,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:00,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:41:00,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:41:04,262 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:04,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:41:09,857 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:41:09,862 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:14,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:41:15,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:41:15,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:41:15,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:21,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:24,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:41:27,178 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:30,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:41:30,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:41:32,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:34,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:41:38,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:39,863 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:41:44,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:44,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:41:45,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:41:45,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:41:49,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:41:54,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:41:55,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:00,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:42:00,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:42:00,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:00,898 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:42:00,901 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:42:00,907 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:42:00,910 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:42:01,464 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:42:02,468 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:42:04,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:42:06,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:09,869 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:42:11,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:14,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:42:15,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:42:15,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:42:16,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:22,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:24,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:42:28,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:30,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:42:30,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:42:33,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:34,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:42:39,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:39,875 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:42:44,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:42:45,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:42:45,110 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:42:45,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:51,180 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:42:54,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:42:57,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:00,103 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:43:00,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:43:02,273 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:04,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:43:08,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:09,881 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:43:13,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:14,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:43:15,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:43:15,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:43:18,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:24,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:24,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:43:29,181 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:30,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:43:30,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:43:34,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:34,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:43:39,887 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:43:39,888 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:44,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:43:45,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:43:45,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:43:45,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:51,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:43:54,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:43:57,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:00,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:44:00,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:44:02,269 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:04,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:44:08,182 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:09,894 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:44:13,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:14,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:44:15,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:44:15,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:44:18,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:24,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:24,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:44:30,104 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:44:30,109 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:44:30,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:34,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:44:36,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:39,899 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:44:42,183 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:44,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:44:45,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:44:45,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:44:47,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:53,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:44:54,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:44:59,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:00,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:45:00,112 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:45:04,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:04,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:45:09,905 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:45:09,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:14,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:45:15,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:45:15,110 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:45:15,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:21,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:24,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:45:27,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:30,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:45:30,109 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:45:32,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:34,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:45:38,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:39,911 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:45:43,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:44,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:45:45,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:45:45,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:45:48,290 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:54,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:45:54,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:46:00,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:46:00,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:46:00,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:04,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:46:06,185 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:09,916 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:46:12,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:14,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:46:15,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:46:15,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:46:17,272 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:23,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:24,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:46:29,186 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:30,105 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:46:30,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:46:34,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:34,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:46:39,922 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:46:39,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:44,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:46:45,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:46:45,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:46:45,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:51,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:46:54,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:46:57,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:00,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:47:00,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:47:02,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:04,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:47:08,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:09,927 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:47:13,187 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:14,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:47:15,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:47:15,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:47:18,281 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:24,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:24,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:47:29,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:30,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:47:30,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:47:31,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:47:31,918 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:47:31,918 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:47:31,923 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:47:32,861 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:47:34,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:47:34,868 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:47:34,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:39,929 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:47:39,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:44,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:47:45,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:47:45,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:47:45,358 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:51,188 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:47:54,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:47:57,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:00,106 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:48:00,107 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:48:02,297 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:04,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:48:08,189 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:09,935 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:48:13,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:14,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:48:15,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:48:15,108 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:48:18,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:24,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:24,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:48:30,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:48:30,108 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:48:30,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:34,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:48:36,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:39,941 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:48:42,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:44,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:48:45,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:48:45,108 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:48:47,258 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:53,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:48:54,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:48:58,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:00,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:49:00,108 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:49:03,261 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:04,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:49:09,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:09,950 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:49:14,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:49:15,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:49:15,108 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:49:15,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:21,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:24,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:49:27,191 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:30,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:49:30,109 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:49:32,274 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:34,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:49:38,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:39,953 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:49:44,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:44,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:49:45,107 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:49:45,137 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:49:49,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:49:54,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:49:55,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:00,193 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:01,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:50:01,149 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:50:04,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:50:05,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:09,958 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:50:10,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:14,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:50:16,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:50:16,147 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:50:16,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:22,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:24,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:50:28,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:31,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:50:31,147 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:50:33,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:34,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:50:39,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:39,962 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:50:44,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:44,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:50:46,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:50:46,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:50:49,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:50:54,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:50:55,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:01,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:51:01,147 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:51:01,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:04,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:51:07,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:09,967 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:51:13,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:14,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:51:16,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:51:16,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:51:18,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:24,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:24,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:51:30,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:31,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:51:31,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:51:34,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:51:35,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:39,971 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:51:40,974 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:44,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:51:46,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:51:46,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:51:46,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:52,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:51:54,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:51:57,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:01,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:52:01,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:52:02,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:04,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:52:08,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:09,980 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:52:14,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:14,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:52:16,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:52:16,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:52:19,362 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:24,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:52:25,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:31,147 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:52:31,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:52:31,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:34,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:52:37,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:39,985 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:52:42,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:44,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:52:46,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:52:46,148 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:52:47,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:53,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:52:54,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:52:59,153 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:01,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:53:01,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:53:04,029 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:53:04,034 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:53:04,035 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:53:04,038 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:53:04,138 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:53:04,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:53:05,040 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:07,147 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:53:09,991 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:53:10,994 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:14,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:53:16,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:53:16,149 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:53:16,387 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:22,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:24,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:53:28,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:31,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:53:31,149 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:53:33,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:34,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:53:39,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:39,994 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:53:44,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:53:45,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:46,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:53:46,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:53:50,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:53:54,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:53:56,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:01,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:54:01,149 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:54:01,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:04,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:54:07,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:10,006 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:54:12,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:14,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:54:16,148 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:54:16,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:54:17,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:23,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:24,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:54:29,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:31,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:54:31,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:54:34,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:34,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:54:40,006 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:54:40,012 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:44,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:54:45,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:46,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:54:46,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:54:50,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:54:54,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:54:56,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:01,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:55:01,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:55:01,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:04,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:55:07,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:10,011 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:55:13,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:14,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:55:16,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:55:16,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:55:18,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:24,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:24,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:55:29,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:31,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:55:31,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:55:34,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:34,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:55:40,018 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:55:40,022 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:44,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:55:45,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:46,149 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:55:46,150 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:55:50,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:55:54,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:55:56,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:01,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:56:01,151 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:56:01,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:04,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:56:07,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:10,024 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:56:13,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:14,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:56:16,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:56:16,151 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:56:18,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:24,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:24,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:56:29,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:31,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:56:31,151 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:56:34,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:34,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:56:40,029 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:56:40,034 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:44,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:56:45,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:46,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:56:46,151 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:56:50,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:56:54,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:56:56,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:01,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:57:01,152 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:57:01,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:04,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:57:07,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:10,036 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:57:13,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:14,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:57:16,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:57:16,151 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:57:18,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:24,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:24,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:57:30,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:31,150 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:57:31,151 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:57:34,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:57:35,353 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:40,041 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:57:41,043 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:44,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:57:46,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:57:46,152 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:57:46,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:52,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:57:54,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:57:57,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:01,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:58:01,152 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:58:02,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:04,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:58:08,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:10,047 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:58:13,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:14,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:58:16,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:58:16,152 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:58:18,388 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:24,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:24,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:58:30,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:31,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:58:31,152 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:58:34,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:58:35,225 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-02 23:58:35,229 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-02 23:58:35,233 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-02 23:58:35,233 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:35,236 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-02 23:58:35,358 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-02 23:58:37,364 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-02 23:58:40,055 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:58:41,057 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:44,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:58:46,151 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:58:46,152 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:58:46,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:52,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:58:54,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:58:58,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:01,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:59:01,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:59:03,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:04,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:59:09,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:10,058 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:59:14,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:14,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:59:16,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:59:16,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:59:19,392 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:24,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:59:25,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:31,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:59:31,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:59:31,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:34,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:59:37,298 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:40,061 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-02 23:59:42,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:44,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:59:46,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-02 23:59:46,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-02 23:59:47,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:53,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-02 23:59:54,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-02 23:59:58,299 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:01,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:00:01,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:00:03,368 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:04,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:00:09,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:10,069 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:00:14,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:00:15,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:16,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:00:16,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:00:20,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:24,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:00:26,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:31,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:00:31,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:00:31,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:34,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:00:37,300 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:40,075 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:00:42,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:44,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:00:46,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:00:46,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:00:47,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:53,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:00:54,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:00:59,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:01,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:01:01,153 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:01:04,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:04,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:01:10,080 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:01:10,084 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:14,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:01:15,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:16,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:01:16,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:01:20,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:24,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:01:26,301 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:31,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:01:31,154 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:01:31,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:34,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:01:37,302 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:40,084 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:01:42,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:44,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:01:46,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:01:46,155 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:01:47,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:53,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:01:54,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:01:59,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:01,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:02:01,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:02:04,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:04,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:02:10,090 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:02:10,094 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:14,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:02:15,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:16,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:02:16,155 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:02:20,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:24,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:02:26,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:31,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:02:31,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:02:31,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:34,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:02:37,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:40,095 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:02:42,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:44,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:02:46,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:02:46,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:02:48,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:54,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:02:54,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:03:00,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:01,154 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:03:01,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:03:04,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:03:05,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:10,097 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:03:11,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:14,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:03:16,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:03:16,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:03:16,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:22,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:24,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:03:27,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:31,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:03:31,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:03:32,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:34,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:03:38,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:40,107 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:03:44,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:44,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:03:46,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:03:46,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:03:49,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:03:54,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:03:55,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:01,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:04:01,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:04:01,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:04,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:04:07,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:07,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:04:07,499 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:04:07,500 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:04:07,509 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:04:07,894 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:04:08,897 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:04:10,109 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:04:12,311 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:14,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:04:16,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:04:16,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:04:17,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:23,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:24,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:04:29,312 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:31,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:04:31,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:04:34,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:34,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:04:40,115 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:04:40,120 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:44,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:04:45,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:46,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:04:46,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:04:50,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:04:54,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:04:56,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:01,155 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:05:01,156 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:05:01,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:04,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:05:07,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:10,120 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:05:13,313 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:14,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:05:16,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:05:16,157 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:05:18,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:24,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:24,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:05:30,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:31,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:05:31,157 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:05:34,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:05:35,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:40,125 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:05:41,128 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:44,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:05:46,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:05:46,158 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:05:46,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:52,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:05:54,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:05:58,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:01,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:06:01,158 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:06:04,314 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:04,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:06:09,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:10,130 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:06:14,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:06:15,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:16,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:06:16,157 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:06:20,355 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:24,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:06:26,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:31,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:06:31,157 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:06:31,330 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:34,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:06:37,315 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:40,135 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:06:42,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:44,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:06:46,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:06:46,158 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:06:47,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:53,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:06:54,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:06:58,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:01,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:07:01,157 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:07:03,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:04,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:07:09,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:10,141 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:07:14,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:07:15,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:16,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:07:16,158 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:07:20,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:24,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:07:26,316 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:31,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:07:31,158 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:07:31,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:34,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:07:37,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:40,146 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:07:42,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:44,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:07:46,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:07:46,158 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:07:47,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:53,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:07:54,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:07:59,318 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:01,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:08:01,159 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:08:04,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:04,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:08:10,151 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:08:10,157 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:14,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:08:15,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:16,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:08:16,159 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:08:20,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:24,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:08:26,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:31,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:08:31,159 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:08:31,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:34,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:08:37,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:40,159 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:08:42,319 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:44,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:08:46,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:08:46,159 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:08:47,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:53,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:08:54,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:08:59,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:01,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:09:01,159 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:09:04,385 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:04,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:09:10,163 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:09:10,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:14,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:09:15,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:16,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:09:16,160 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:09:20,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:24,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:09:26,320 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:31,158 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:09:31,159 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:09:31,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:34,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:09:37,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:39,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:09:39,691 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:09:39,691 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:09:39,695 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:09:40,167 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:09:40,217 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:09:41,220 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:09:43,321 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:44,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:09:46,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:09:46,159 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:09:48,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:54,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:09:54,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:10:00,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:01,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:10:01,160 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:10:04,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:10:05,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:10,214 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:10:11,217 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:14,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:10:16,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:10:16,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:10:16,371 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:22,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:24,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:10:28,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:31,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:10:31,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:10:33,391 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:34,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:10:39,322 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:40,221 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:10:44,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:10:45,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:46,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:10:46,160 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:10:50,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:10:54,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:10:56,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:01,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:11:01,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:11:01,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:04,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:11:07,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:10,226 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:11:12,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:14,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:11:16,159 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:11:16,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:11:17,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:23,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:24,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:11:29,324 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:31,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:11:31,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:11:34,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:34,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:11:40,231 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:11:40,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:44,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:11:45,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:46,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:11:46,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:11:50,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:11:54,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:11:56,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:01,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:12:01,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:12:01,327 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:04,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:12:07,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:10,236 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:12:13,325 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:14,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:12:16,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:12:16,162 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:12:18,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:24,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:24,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:12:30,326 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:31,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:12:31,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:12:34,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:12:35,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:40,243 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:12:41,245 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:44,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:12:46,160 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:12:46,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:12:46,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:52,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:12:54,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:12:58,331 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:01,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:13:01,161 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:13:03,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:04,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:13:09,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:10,247 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:13:14,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:13:15,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:16,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:13:16,162 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:13:21,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:24,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:13:27,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:31,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:13:31,162 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:13:33,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:34,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:13:38,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:40,254 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:13:44,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:44,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:13:46,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:13:46,162 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:13:49,370 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:13:54,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:13:55,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:00,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:01,161 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:14:01,163 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:14:04,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:14:06,333 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:10,258 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:14:11,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:14,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:14:16,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:14:16,163 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:14:16,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:22,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:24,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:14:28,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:31,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:14:31,163 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:14:33,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:34,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:14:39,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:40,263 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:14:44,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:44,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:14:46,162 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:14:46,163 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:14:50,334 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:14:54,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:14:55,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:00,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:01,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:15:01,164 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:15:04,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:15:06,335 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:10,266 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:15:11,126 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:15:11,129 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:15:11,131 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:15:11,136 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:15:11,733 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:15:12,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:13,760 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:15:14,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:15:16,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:15:16,164 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:15:17,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:23,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:24,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:15:29,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:31,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:15:31,165 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:15:34,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:34,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:15:40,271 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:15:40,276 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:44,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:15:45,337 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:46,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:15:46,165 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:15:50,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:15:54,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:15:56,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:01,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:16:01,164 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:16:01,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:04,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:16:07,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:10,275 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:16:13,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:14,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:16:16,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:16:16,164 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:16:18,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:24,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:24,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:16:30,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:31,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:16:31,164 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:16:34,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:16:35,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:40,279 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:16:41,282 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:44,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:16:46,163 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:16:46,164 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:16:46,389 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:52,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:16:54,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:16:58,339 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:01,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:17:01,165 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:17:03,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:04,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:17:09,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:10,286 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:17:14,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:17:15,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:16,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:17:16,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:17:21,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:24,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:17:27,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:31,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:17:31,165 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:17:33,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:34,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:17:38,340 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:40,292 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:17:44,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:44,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:17:46,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:17:46,165 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:17:50,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:17:54,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:17:56,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:01,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:18:01,165 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:18:01,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:04,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:18:07,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:10,300 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:18:13,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:14,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:18:16,164 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:18:16,165 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:18:18,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:24,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:24,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:18:30,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:31,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:18:31,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:18:34,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:18:35,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:40,304 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:18:41,307 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:44,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:18:46,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:18:46,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:18:46,341 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:51,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:18:54,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:18:56,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:01,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:19:01,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:19:01,356 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:04,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:19:07,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:10,312 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:19:12,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:14,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:19:16,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:19:16,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:19:18,343 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:24,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:24,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:19:29,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:31,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:19:31,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:19:34,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:19:35,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:40,314 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:19:41,317 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:44,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:19:46,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:19:46,167 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:19:46,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:52,344 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:19:54,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:19:57,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:01,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:20:01,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:20:04,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:20:04,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:10,319 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:20:10,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:14,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:20:15,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:16,165 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:20:16,166 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:20:20,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:24,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:20:26,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:31,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:20:31,167 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:20:31,374 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:34,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:20:37,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:40,325 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:20:43,346 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:43,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:20:43,659 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:20:43,660 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:20:43,663 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:20:44,232 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:20:44,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:20:46,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:20:46,167 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:20:46,241 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:20:48,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:54,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:20:54,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:20:59,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:01,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:21:01,167 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:21:04,366 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:04,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:21:10,331 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:21:10,336 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:14,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:21:15,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:16,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:21:16,167 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:21:20,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:24,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:21:26,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:31,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:21:31,167 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:21:31,364 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:34,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:21:37,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:40,337 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:21:43,348 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:44,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:21:46,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:21:46,168 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:21:48,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:54,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:21:54,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:22:00,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:01,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:22:01,168 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:22:04,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:22:05,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:10,342 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:22:11,345 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:14,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:22:16,166 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:22:16,167 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:22:16,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:22,349 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:24,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:22:27,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:31,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:22:31,169 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:22:32,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:34,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:22:38,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:40,343 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:22:44,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:44,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:22:46,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:22:46,168 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:22:50,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:22:54,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:22:55,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:01,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:23:01,168 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:23:01,338 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:04,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:23:06,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:10,352 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:23:12,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:14,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:23:16,167 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:23:16,169 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:23:17,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:23,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:24,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:23:29,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:31,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:23:31,169 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:23:34,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:34,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:23:40,357 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:23:40,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:44,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:23:46,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:23:46,169 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:23:46,347 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:51,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:23:54,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:23:57,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:01,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:24:01,169 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:24:02,372 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:04,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:24:08,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:10,362 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:24:14,353 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:14,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:24:16,168 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:24:16,169 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:24:20,353 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:24,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:24:26,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:31,169 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:24:31,170 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:24:31,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:34,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:24:37,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:40,367 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:24:43,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:44,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:24:46,169 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:24:46,170 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:24:48,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:54,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:24:54,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:25:00,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:01,169 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:25:01,170 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:25:04,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:25:06,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:10,373 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:25:12,354 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:14,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:25:16,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:25:16,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:25:17,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:23,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:24,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:25:29,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:31,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:25:31,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:25:34,581 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:34,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:25:40,379 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:25:40,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:44,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:25:45,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:46,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:25:46,426 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:25:50,579 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:25:54,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:25:56,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:01,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:26:01,426 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:26:01,574 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:04,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:26:07,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:10,386 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:26:13,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:14,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:26:15,825 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:26:15,829 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:26:15,830 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:26:15,835 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:26:16,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:26:16,426 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:26:16,557 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:26:18,563 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:26:18,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:24,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:24,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:26:30,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:31,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:26:31,426 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:26:34,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:26:35,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:40,391 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:26:41,394 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:44,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:26:46,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:26:46,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:26:46,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:52,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:26:54,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:26:57,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:01,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:27:01,426 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:27:02,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:04,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:27:08,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:10,397 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:27:14,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:14,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:27:16,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:27:16,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:27:19,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:24,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:27:25,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:30,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:31,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:27:31,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:27:34,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:27:35,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:40,403 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:27:41,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:44,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:27:46,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:27:46,428 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:27:46,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:52,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:27:54,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:27:58,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:01,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:28:01,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:28:03,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:04,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:28:09,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:10,410 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:28:14,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:28:15,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:16,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:28:16,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:28:20,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:24,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:28:26,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:31,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:28:31,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:28:31,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:34,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:28:37,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:40,414 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:28:43,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:44,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:28:46,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:28:46,427 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:28:48,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:54,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:28:54,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:28:59,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:01,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:29:01,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:29:04,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:04,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:29:10,418 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:29:10,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:14,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:29:15,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:16,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:29:16,428 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:29:20,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:24,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:29:26,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:31,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:29:31,428 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:29:31,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:34,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:29:37,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:40,425 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:29:43,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:44,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:29:46,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:29:46,428 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:29:48,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:54,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:29:54,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:30:00,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:01,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:30:01,428 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:30:04,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:30:05,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:10,432 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:30:11,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:14,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:30:16,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:30:16,429 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:30:16,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:22,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:24,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:30:27,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:31,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:30:31,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:30:32,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:34,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:30:38,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:40,438 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:30:44,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:44,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:30:46,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:30:46,429 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:30:49,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:30:54,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:30:55,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:00,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:01,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:31:01,429 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:31:04,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:31:05,595 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:10,443 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:31:11,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:14,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:31:16,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:31:16,429 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:31:16,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:22,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:24,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:31:28,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:31,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:31:31,429 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:31:33,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:34,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:31:39,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:40,447 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:31:44,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:44,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:31:46,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:31:46,430 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:31:46,986 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:31:46,990 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:31:46,991 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:31:46,994 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:31:47,828 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:31:47,829 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:31:49,996 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:31:54,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:31:55,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:00,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:01,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:32:01,430 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:32:04,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:32:05,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:10,454 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:32:11,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:14,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:32:16,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:32:16,430 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:32:16,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:22,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:24,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:32:28,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:31,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:32:31,430 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:32:33,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:34,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:32:39,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:40,460 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:32:44,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:32:45,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:46,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:32:46,430 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:32:50,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:32:54,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:32:56,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:01,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:33:01,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:33:01,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:04,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:33:07,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:10,467 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:33:12,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:14,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:33:16,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:33:16,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:33:17,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:23,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:24,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:33:29,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:31,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:33:31,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:33:34,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:34,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:33:40,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:40,472 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:33:44,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:33:46,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:33:46,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:33:46,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:52,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:33:54,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:33:58,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:01,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:34:01,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:34:03,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:04,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:34:09,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:10,478 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:34:14,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:34:15,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:16,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:34:16,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:34:20,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:24,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:34:26,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:31,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:34:31,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:34:31,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:34,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:34:37,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:40,484 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:34:43,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:44,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:34:46,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:34:46,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:34:48,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:54,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:34:54,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:35:00,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:01,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:35:01,431 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:35:04,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:35:05,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:10,490 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:35:11,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:14,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:35:16,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:35:16,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:35:16,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:22,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:24,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:35:28,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:31,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:35:31,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:35:33,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:34,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:35:39,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:40,495 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:35:44,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:44,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:35:46,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:35:46,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:35:49,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:35:54,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:35:55,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:01,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:36:01,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:36:01,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:04,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:36:07,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:10,501 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:36:13,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:14,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:36:16,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:36:16,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:36:18,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:24,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:24,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:36:30,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:31,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:36:31,432 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:36:34,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:36:35,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:40,506 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:36:41,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:44,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:36:46,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:36:46,433 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:36:46,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:52,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:36:54,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:36:58,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:01,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:37:01,433 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:37:03,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:04,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:37:09,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:10,512 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:37:14,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:37:15,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:16,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:37:16,433 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:37:19,241 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:37:19,245 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:37:19,245 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:37:19,249 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:37:20,141 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:37:20,142 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:37:21,251 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:24,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:37:26,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:31,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:37:31,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:37:31,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:34,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:37:37,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:40,520 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:37:42,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:44,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:37:46,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:37:46,433 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:37:47,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:53,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:37:54,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:37:58,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:01,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:38:01,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:38:03,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:04,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:38:09,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:10,524 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:38:14,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:38:15,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:16,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:38:16,433 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:38:20,584 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:24,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:38:26,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:31,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:38:31,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:38:31,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:34,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:38:37,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:40,530 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:38:43,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:44,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:38:46,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:38:46,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:38:48,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:54,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:38:54,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:39:00,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:01,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:39:01,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:39:04,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:39:05,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:10,535 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:39:11,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:14,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:39:16,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:39:16,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:39:16,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:22,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:24,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:39:28,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:31,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:39:31,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:39:33,602 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:34,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:39:39,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:40,541 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:39:44,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:44,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:39:46,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:39:46,434 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:39:49,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:39:54,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:39:55,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:01,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:40:01,435 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:40:01,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:04,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:40:07,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:10,547 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:40:13,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:14,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:40:16,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:40:16,435 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:40:18,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:24,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:24,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:40:29,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:31,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:40:31,436 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:40:34,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:40:34,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:40,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:40,553 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:40:44,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:40:46,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:40:46,436 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:40:46,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:52,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:40:54,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:40:57,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:01,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:41:01,435 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:41:02,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:04,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:41:08,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:10,559 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:41:14,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:14,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:41:16,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:41:16,435 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:41:19,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:24,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:41:25,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:31,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:41:31,436 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:41:31,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:34,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:41:37,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:40,564 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:41:43,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:44,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:41:46,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:41:46,436 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:41:48,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:54,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:41:54,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:42:00,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:01,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:42:01,436 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:42:04,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:42:05,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:10,570 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:42:11,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:14,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:42:16,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:42:16,436 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:42:16,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:22,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:24,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:42:28,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:31,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:42:31,436 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:42:33,586 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:34,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:42:39,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:40,574 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:42:44,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:44,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:42:46,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:42:46,437 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:42:49,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:42:50,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:42:50,502 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:42:50,505 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:42:50,509 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:42:50,548 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:42:52,555 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:42:54,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:42:55,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:01,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:43:01,437 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:43:01,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:04,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:43:07,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:10,579 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:43:12,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:14,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:43:16,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:43:16,437 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:43:17,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:23,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:24,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:43:29,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:31,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:43:31,437 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:43:34,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:34,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:43:40,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:40,584 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:43:44,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:43:46,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:43:46,438 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:43:46,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:52,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:43:54,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:43:58,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:01,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:44:01,438 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:44:03,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:04,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:44:09,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:10,590 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:44:14,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:14,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:44:16,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:44:16,438 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:44:19,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:24,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:44:25,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:30,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:31,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:44:31,438 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:44:34,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:44:35,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:40,596 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:44:41,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:44,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:44:46,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:44:46,438 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:44:46,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:52,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:44:54,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:44:58,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:01,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:45:01,437 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:45:03,601 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:04,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:45:09,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:10,602 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:45:14,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:14,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:45:16,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:45:16,438 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:45:19,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:24,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:45:25,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:31,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:45:31,439 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:45:31,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:34,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:45:37,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:40,608 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:45:42,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:44,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:45:46,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:45:46,439 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:45:47,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:53,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:45:54,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:45:59,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:01,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:46:01,439 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:46:04,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:04,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:46:10,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:10,614 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:46:14,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:46:15,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:16,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:46:16,439 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:46:20,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:24,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:46:26,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:31,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:46:31,439 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:46:31,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:34,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:46:37,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:40,621 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:46:42,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:44,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:46:46,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:46:46,439 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:46:47,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:53,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:46:54,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:46:59,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:01,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:47:01,440 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:47:04,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:04,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:47:10,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:10,623 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:47:14,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:47:16,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:47:16,441 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:47:16,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:22,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:24,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:47:28,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:31,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:47:31,440 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:47:33,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:34,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:47:39,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:40,630 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:47:44,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:47:45,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:46,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:47:46,440 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:47:50,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:47:54,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:47:56,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:01,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:48:01,440 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:48:01,589 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:04,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:48:07,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:10,634 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:48:12,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:14,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:48:16,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:48:16,441 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:48:17,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:22,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:48:22,673 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:48:22,674 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:48:22,681 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:48:22,993 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:48:23,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:24,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:48:24,999 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:48:29,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:31,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:48:31,441 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:48:34,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:34,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:48:40,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:40,640 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:48:44,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:48:45,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:46,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:48:46,441 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:48:50,611 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:48:54,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:48:56,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:01,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:49:01,441 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:49:01,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:04,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:49:07,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:10,644 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:49:12,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:14,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:49:16,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:49:16,442 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:49:17,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:23,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:24,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:49:29,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:31,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:49:31,442 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:49:34,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:34,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:49:40,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:40,650 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:49:44,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:49:46,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:49:46,441 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:49:46,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:52,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:49:54,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:49:57,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:01,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:50:01,441 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:50:02,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:04,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:50:08,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:10,656 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:50:13,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:14,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:50:16,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:50:16,442 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:50:18,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:24,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:24,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:50:30,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:31,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:50:31,442 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:50:34,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:50:35,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:40,662 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:50:40,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:44,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:50:46,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:50:46,442 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:50:46,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:52,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:50:54,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:50:58,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:01,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:51:01,442 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:51:03,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:04,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:51:09,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:10,668 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:51:14,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:51:15,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:16,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:51:16,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:51:20,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:24,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:51:26,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:31,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:51:31,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:51:31,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:34,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:51:37,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:40,674 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:51:43,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:44,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:51:46,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:51:46,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:51:48,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:54,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:51:54,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:52:00,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:01,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:52:01,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:52:04,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:52:05,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:10,680 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:52:10,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:14,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:52:16,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:52:16,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:52:16,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:22,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:24,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:52:28,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:31,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:52:31,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:52:33,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:34,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:52:39,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:40,686 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:52:44,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:44,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:52:46,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:52:46,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:52:49,603 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:52:54,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:52:55,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:01,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:53:01,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:53:01,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:04,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:53:07,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:10,690 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:53:12,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:14,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:53:16,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:53:16,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:53:17,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:23,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:24,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:53:29,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:31,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:53:31,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:53:34,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:53:34,882 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:40,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:40,694 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:53:44,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:53:46,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:53:46,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:53:46,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:52,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:53:54,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:53:54,860 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:53:54,864 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:53:54,866 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:53:54,872 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:53:55,227 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:53:57,233 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:53:58,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:01,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:54:01,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:54:03,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:04,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:54:09,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:10,700 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:54:14,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:54:15,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:16,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:54:16,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:54:20,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:24,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:54:26,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:31,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:54:31,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:54:31,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:34,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:54:37,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:40,706 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:54:43,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:44,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:54:46,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:54:46,445 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:54:48,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:54,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:54:54,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:55:00,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:01,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:55:01,444 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:55:04,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:55:05,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:10,711 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:55:10,716 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:14,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:55:16,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:55:16,445 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:55:16,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:22,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:24,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:55:28,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:31,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:55:31,445 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:55:33,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:34,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:55:39,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:40,717 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:55:44,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:55:45,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:46,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:55:46,445 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:55:50,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:55:54,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:55:56,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:01,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:56:01,445 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:56:01,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:04,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:56:07,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:10,723 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:56:13,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:14,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:56:16,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:56:16,445 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:56:18,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:24,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:24,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:56:30,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:31,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:56:31,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:56:34,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:56:35,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:40,729 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:56:40,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:44,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:56:46,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:56:46,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:56:46,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:52,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:56:54,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:56:58,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:01,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:57:01,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:57:03,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:04,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:57:09,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:10,737 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:57:14,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:14,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:57:16,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:57:16,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:57:19,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:24,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:57:25,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:31,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:57:31,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:57:31,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:34,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:57:37,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:40,740 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:57:43,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:44,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:57:46,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:57:46,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:57:48,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:54,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:57:54,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:58:00,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:01,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:58:01,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:58:04,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:58:05,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:10,747 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:58:10,751 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:14,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:58:16,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:58:16,447 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:58:16,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:22,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:24,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:58:28,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:31,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:58:31,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:58:33,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:34,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:58:39,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:40,751 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:58:44,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:44,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:58:46,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:58:46,446 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:58:49,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:58:54,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:58:55,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:01,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:59:01,447 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:59:01,604 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:04,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:59:07,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:10,758 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:59:13,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:14,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:59:16,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:59:16,447 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:59:18,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:24,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:24,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:59:25,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 00:59:25,964 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 00:59:25,964 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 00:59:25,968 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 00:59:26,457 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 00:59:27,460 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 00:59:29,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:31,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:59:31,447 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:59:34,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:34,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:59:40,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:40,763 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 00:59:44,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:59:46,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 00:59:46,447 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 00:59:46,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:52,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 00:59:54,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 00:59:58,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:01,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:00:01,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:00:03,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:04,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:00:09,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:10,772 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:00:14,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:00:15,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:16,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:00:16,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:00:20,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:24,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:00:26,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:31,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:00:31,447 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:00:31,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:34,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:00:37,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:40,775 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:00:43,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:44,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:00:46,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:00:46,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:00:48,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:54,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:00:54,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:01:00,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:01,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:01:01,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:01:04,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:01:05,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:10,779 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:01:10,783 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:14,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:01:16,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:01:16,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:01:16,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:22,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:24,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:01:27,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:31,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:01:31,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:01:34,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:01:35,137 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:40,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:40,784 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:01:44,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:01:45,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:46,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:01:46,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:01:50,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:01:54,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:01:56,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:01,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:02:01,447 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:02:01,599 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:04,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:02:07,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:10,792 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:02:13,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:14,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:02:16,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:02:16,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:02:18,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:24,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:24,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:02:30,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:31,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:02:31,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:02:34,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:02:35,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:40,797 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:02:40,802 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:44,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:02:46,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:02:46,449 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:02:46,594 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:52,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:02:54,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:02:57,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:01,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:03:01,449 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:03:02,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:04,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:03:08,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:10,802 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:03:14,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:14,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:03:16,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:03:16,449 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:03:19,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:24,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:03:25,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:31,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:03:31,449 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:03:31,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:34,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:03:37,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:40,809 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:03:43,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:44,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:03:46,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:03:46,450 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:03:48,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:54,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:03:54,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:04:00,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:01,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:04:01,450 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:04:04,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:04:05,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:10,815 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:04:10,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:14,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:04:16,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:04:16,450 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:04:16,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:22,496 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:24,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:04:28,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:31,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:04:31,450 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:04:33,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:34,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:04:39,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:40,821 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:04:44,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:44,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:04:46,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:04:46,450 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:04:49,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:54,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:04:55,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:04:58,102 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:04:58,108 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:04:58,109 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:04:58,113 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:04:58,776 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:04:59,780 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:05:01,144 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:01,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:05:01,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:05:04,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:05:06,497 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:10,827 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:05:11,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:14,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:05:16,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:05:16,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:05:16,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:22,498 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:24,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:05:28,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:31,449 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:05:31,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:05:33,711 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:34,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:05:39,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:40,833 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:05:44,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:44,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:05:46,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:05:46,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:05:49,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:05:54,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:05:55,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:01,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:06:01,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:06:01,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:04,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:06:07,499 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:10,839 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:06:12,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:14,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:06:16,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:06:16,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:06:17,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:23,500 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:24,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:06:29,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:31,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:06:31,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:06:34,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:34,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:06:40,501 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:40,844 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:06:44,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:06:46,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:06:46,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:06:46,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:52,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:06:54,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:06:58,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:01,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:07:01,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:07:03,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:04,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:07:09,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:10,852 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:07:14,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:14,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:07:16,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:07:16,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:07:19,720 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:24,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:07:25,502 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:31,450 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:07:31,451 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:07:31,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:34,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:07:37,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:40,856 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:07:42,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:44,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:07:46,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:07:46,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:07:47,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:53,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:07:54,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:07:59,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:01,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:08:01,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:08:04,598 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:04,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:08:10,503 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:10,861 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:08:14,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:08:15,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:16,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:08:16,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:08:20,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:24,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:08:26,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:31,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:08:31,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:08:31,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:34,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:08:37,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:40,867 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:08:43,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:44,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:08:46,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:08:46,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:08:48,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:54,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:08:54,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:09:00,504 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:01,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:09:01,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:09:04,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:09:05,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:10,871 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:09:10,876 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:14,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:09:16,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:09:16,453 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:09:16,596 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:22,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:24,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:09:27,505 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:31,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:09:31,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:09:32,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:34,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:09:38,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:40,878 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:09:44,506 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:44,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:09:46,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:09:46,453 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:09:49,607 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:09:54,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:09:55,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:01,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:10:01,453 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:10:01,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:04,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:10:07,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:10,883 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:10:12,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:14,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:10:16,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:10:16,453 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:10:17,608 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:23,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:24,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:10:29,507 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:30,308 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:10:30,311 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:10:30,312 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:10:30,315 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:10:31,054 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:10:31,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:10:31,453 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:10:32,057 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:10:34,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:34,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:10:40,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:40,889 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:10:44,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:10:46,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:10:46,454 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:10:46,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:52,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:10:54,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:10:58,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:01,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:11:01,453 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:11:03,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:04,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:11:09,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:10,895 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:11:14,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:11:15,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:16,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:11:16,454 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:11:20,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:24,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:11:26,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:31,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:11:31,454 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:11:31,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:34,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:11:37,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:40,901 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:11:42,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:44,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:11:46,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:11:46,454 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:11:47,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:53,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:11:54,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:11:59,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:01,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:12:01,455 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:12:04,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:04,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:12:10,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:10,907 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:12:14,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:12:16,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:12:16,455 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:12:16,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:22,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:24,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:12:27,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:31,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:12:31,455 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:12:32,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:34,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:12:38,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:40,911 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:12:44,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:44,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:12:46,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:12:46,455 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:12:49,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:12:54,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:12:55,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:00,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:01,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:13:01,455 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:13:04,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:13:05,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:10,917 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:13:10,922 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:14,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:13:16,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:13:16,455 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:13:16,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:22,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:24,706 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:13:28,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:31,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:13:31,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:13:33,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:34,706 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:13:39,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:40,922 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:13:44,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:13:45,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:46,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:13:46,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:13:50,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:13:54,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:13:56,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:01,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:14:01,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:14:01,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:04,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:14:07,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:10,924 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:14:12,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:14,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:14:16,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:14:16,457 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:14:17,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:23,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:24,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:14:29,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:31,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:14:31,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:14:34,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:34,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:14:40,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:40,931 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:14:44,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:14:46,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:14:46,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:14:46,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:52,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:14:54,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:14:58,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:01,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:15:01,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:15:03,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:04,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:15:09,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:10,935 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:15:14,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:14,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:15:16,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:15:16,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:15:19,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:24,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:15:25,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:30,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:31,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:15:31,456 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:15:34,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:15:35,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:40,941 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:15:40,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:44,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:15:46,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:15:46,457 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:15:46,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:52,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:15:54,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:15:58,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:01,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:16:01,416 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:16:01,417 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:16:01,421 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:16:01,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:16:01,457 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:16:02,361 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:16:03,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:04,368 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:16:04,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:16:09,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:10,943 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:16:14,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:16:15,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:16,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:16:16,457 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:16:20,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:24,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:16:26,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:31,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:16:31,457 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:16:31,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:34,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:16:37,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:40,951 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:16:42,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:44,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:16:46,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:16:46,458 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:16:47,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:53,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:16:54,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:16:59,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:01,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:17:01,458 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:17:04,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:04,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:17:10,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:10,956 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:17:14,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:17:16,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:17:16,458 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:17:16,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:22,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:24,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:17:28,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:31,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:17:31,458 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:17:33,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:34,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:17:39,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:40,962 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:17:44,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:17:45,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:46,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:17:46,459 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:17:50,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:17:54,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:17:56,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:01,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:18:01,459 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:18:01,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:04,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:18:07,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:10,968 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:18:12,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:15,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:18:16,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:18:16,459 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:18:17,617 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:23,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:25,899 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:18:28,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:31,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:18:31,459 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:18:33,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:35,899 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:18:39,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:40,974 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:18:44,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:45,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:18:46,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:18:46,459 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:18:49,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:55,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:18:55,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:19:00,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:01,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:19:01,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:19:05,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:19:06,156 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:10,980 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:19:11,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:15,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:19:16,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:19:16,459 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:19:16,722 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:22,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:25,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:19:28,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:31,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:19:31,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:19:33,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:35,900 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:19:39,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:40,986 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:19:44,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:45,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:19:46,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:19:46,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:19:49,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:55,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:19:55,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:20:00,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:01,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:20:01,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:20:05,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:05,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:20:10,990 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:20:10,994 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:15,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:20:16,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:20:16,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:20:16,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:22,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:25,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:20:27,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:31,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:20:31,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:20:32,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:35,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:20:38,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:40,998 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:20:43,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:45,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:20:46,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:20:46,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:20:48,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:54,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:20:55,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:20:59,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:01,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:21:01,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:21:04,716 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:05,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:21:10,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:11,000 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:21:15,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:15,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:21:16,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:21:16,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:21:20,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:25,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:21:26,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:31,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:21:31,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:21:33,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:21:34,896 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:21:34,896 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:34,897 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:21:34,907 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:21:35,669 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:21:35,670 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:21:35,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:21:40,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:41,014 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:21:45,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:45,902 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:21:46,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:21:46,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:21:50,610 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:21:55,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:21:56,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:01,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:22:01,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:22:01,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:05,903 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:22:07,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:11,011 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:22:12,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:15,903 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:22:16,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:22:16,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:22:17,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:23,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:25,903 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:22:29,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:31,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:22:31,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:22:34,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:35,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:22:40,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:41,020 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:22:45,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:22:46,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:22:46,462 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:22:46,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:52,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:22:55,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:22:58,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:01,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:23:01,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:23:03,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:05,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:23:09,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:11,022 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:23:15,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:15,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:23:16,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:23:16,462 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:23:20,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:25,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:23:26,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:31,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:23:31,462 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:23:31,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:35,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:23:37,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:41,028 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:23:43,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:45,904 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:23:46,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:23:46,462 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:23:48,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:54,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:23:55,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:23:59,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:01,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:24:01,462 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:24:04,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:05,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:24:10,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:11,034 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:24:15,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:24:16,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:24:16,462 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:24:16,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:22,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:25,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:24:28,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:31,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:24:31,462 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:24:33,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:35,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:24:39,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:41,039 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:24:45,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:45,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:24:46,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:24:46,463 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:24:50,715 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:24:55,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:24:56,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:01,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:25:01,463 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:25:01,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:05,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:25:07,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:11,044 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:25:13,539 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:15,905 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:25:16,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:25:16,463 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:25:18,621 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:24,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:25,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:25:30,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:31,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:25:31,463 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:25:35,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:35,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:25:41,051 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:25:41,055 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:45,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:25:46,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:25:46,463 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:25:46,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:52,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:25:55,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:25:58,540 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:01,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:26:01,463 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:26:03,622 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:05,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:26:09,541 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:11,056 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:26:15,541 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:15,906 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:26:16,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:26:16,463 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:26:20,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:25,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:26:26,541 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:31,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:26:31,464 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:26:31,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:35,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:26:37,541 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:41,064 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:26:42,542 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:45,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:26:46,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:26:46,464 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:26:47,616 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:53,542 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:26:55,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:26:58,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:01,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:27:01,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:27:03,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:05,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:27:05,637 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:27:05,637 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:27:05,640 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:27:05,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:27:05,984 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:27:07,992 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:27:09,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:11,066 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:27:15,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:15,907 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:27:16,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:27:16,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:27:20,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:25,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:27:26,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:31,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:27:31,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:27:31,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:35,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:27:37,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:41,073 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:27:43,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:45,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:27:46,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:27:46,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:27:48,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:54,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:27:55,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:28:00,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:01,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:28:01,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:28:05,620 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:05,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:28:11,077 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:28:11,082 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:15,908 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:28:16,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:28:16,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:28:16,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:22,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:25,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:28:28,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:31,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:28:31,466 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:28:33,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:35,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:28:39,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:41,080 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:28:44,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:45,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:28:46,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:28:46,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:28:49,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:55,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:28:55,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:29:00,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:01,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:29:01,465 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:29:05,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:05,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:29:11,085 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:29:11,090 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:15,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:29:16,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:29:16,466 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:29:16,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:22,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:25,909 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:29:28,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:31,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:29:31,466 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:29:33,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:35,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:29:39,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:41,091 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:29:45,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:45,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:29:46,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:29:46,466 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:29:50,718 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:29:55,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:29:56,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:01,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:30:01,467 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:30:01,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:05,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:30:07,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:11,095 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:30:12,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:15,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:30:16,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:30:16,466 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:30:17,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:23,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:25,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:30:29,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:31,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:30:31,466 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:30:34,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:35,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:30:40,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:41,101 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:30:45,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:45,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:30:46,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:30:46,467 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:30:50,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:30:55,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:30:56,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:01,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:31:01,467 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:31:01,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:05,910 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:31:07,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:11,107 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:31:13,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:15,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:31:16,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:31:16,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:31:18,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:24,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:25,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:31:30,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:31,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:31:31,467 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:31:35,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:35,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:31:41,111 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:31:41,116 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:45,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:31:46,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:31:46,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:31:46,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:52,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:31:55,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:31:58,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:01,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:32:01,467 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:32:03,711 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:05,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:32:09,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:11,114 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:32:14,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:15,912 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:32:16,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:32:16,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:32:19,713 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:25,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:25,912 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:32:31,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:32:31,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:32:31,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:35,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:32:36,840 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:32:36,844 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:32:36,845 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:32:36,846 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:36,852 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:32:37,472 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:32:38,476 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:32:41,122 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:32:42,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:45,912 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:32:46,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:32:46,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:32:47,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:53,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:32:55,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:32:59,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:01,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:33:01,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:33:04,718 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:05,912 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:33:10,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:11,128 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:33:15,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:15,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:33:16,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:33:16,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:33:20,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:25,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:33:26,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:31,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:33:31,468 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:33:31,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:35,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:33:37,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:41,134 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:33:43,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:45,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:33:46,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:33:46,469 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:33:48,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:54,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:33:55,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:34:00,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:01,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:34:01,469 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:34:05,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:05,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:34:11,139 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:34:11,146 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:15,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:34:16,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:34:16,469 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:34:16,721 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:22,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:25,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:34:28,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:31,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:34:31,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:34:33,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:35,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:34:39,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:41,145 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:34:45,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:45,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:34:46,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:34:46,469 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:34:50,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:34:55,914 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:34:56,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:01,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:35:01,469 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:35:01,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:05,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:35:07,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:11,151 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:35:12,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:15,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:35:16,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:35:16,469 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:35:17,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:23,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:25,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:35:29,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:31,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:35:31,470 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:35:34,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:35,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:35:40,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:41,157 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:35:45,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:35:46,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:35:46,470 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:35:46,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:52,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:35:55,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:35:58,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:01,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:36:01,470 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:36:03,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:05,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:36:09,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:11,164 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:36:15,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:15,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:36:16,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:36:16,470 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:36:20,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:25,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:36:26,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:31,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:36:31,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:36:31,724 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:35,915 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:36:37,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:41,168 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:36:43,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:45,916 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:36:46,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:36:46,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:36:48,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:54,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:36:55,916 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:37:00,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:01,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:37:01,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:37:05,731 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:05,916 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:37:11,174 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:37:11,179 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:15,916 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:37:16,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:37:16,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:37:16,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:22,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:25,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:37:28,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:31,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:37:31,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:37:33,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:35,916 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:37:39,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:41,180 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:37:44,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:45,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:37:46,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:37:46,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:37:49,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:55,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:37:55,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:38:01,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:38:01,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:38:01,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:05,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:38:07,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:09,091 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:38:09,095 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:38:09,096 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:38:09,099 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:38:09,716 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:38:10,718 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:38:11,186 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:38:13,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:15,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:38:16,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:38:16,471 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:38:18,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:24,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:25,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:38:30,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:31,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:38:31,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:38:35,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:35,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:38:41,191 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:38:41,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:45,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:38:46,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:38:46,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:38:46,706 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:52,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:38:55,917 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:38:58,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:01,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:39:01,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:39:03,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:05,918 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:39:09,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:11,197 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:39:15,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:15,918 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:39:16,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:39:16,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:39:20,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:25,918 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:39:26,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:31,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:39:31,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:39:31,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:35,918 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:39:37,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:41,203 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:39:43,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:45,918 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:39:46,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:39:46,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:39:48,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:54,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:39:55,919 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:40:00,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:01,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:40:01,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:40:05,647 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:05,919 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:40:11,209 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:40:11,214 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:15,918 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:40:16,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:40:16,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:40:16,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:22,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:25,919 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:40:28,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:31,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:40:31,473 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:40:33,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:35,919 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:40:39,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:41,215 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:40:44,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:45,919 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:40:46,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:40:46,473 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:40:49,711 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:55,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:40:55,919 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:41:00,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:01,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:41:01,472 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:41:05,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:05,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:41:11,222 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:41:11,226 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:15,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:41:16,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:41:16,474 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:41:16,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:22,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:25,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:41:28,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:31,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:41:31,473 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:41:35,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:35,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:41:40,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:41,226 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:41:45,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:41:46,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:41:46,473 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:41:46,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:52,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:41:55,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:41:58,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:01,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:42:01,474 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:42:03,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:05,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:42:09,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:11,232 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:42:15,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:15,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:42:16,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:42:16,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:42:20,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:25,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:42:26,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:31,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:42:31,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:42:31,624 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:35,920 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:42:37,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:41,238 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:42:43,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:45,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:42:46,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:42:46,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:42:48,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:54,568 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:42:55,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:42:59,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:01,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:43:01,474 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:43:04,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:05,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:43:10,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:11,244 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:43:15,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:15,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:43:16,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:43:16,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:43:20,720 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:25,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:43:26,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:31,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:43:31,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:43:31,720 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:35,922 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:43:37,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:40,184 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:43:40,187 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:43:40,188 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:43:40,191 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:43:41,033 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:43:41,034 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:43:41,250 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:43:43,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:45,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:43:46,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:43:46,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:43:48,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:54,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:43:55,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:44:00,570 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:01,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:44:01,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:44:05,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:05,921 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:44:11,256 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:44:11,260 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:15,922 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:44:16,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:44:16,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:44:16,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:22,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:25,922 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:44:28,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:31,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:44:31,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:44:33,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:35,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:44:39,571 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:41,261 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:44:44,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:45,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:44:46,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:44:46,475 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:44:50,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:44:55,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:44:56,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:01,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:45:01,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:45:01,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:05,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:45:07,648 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:11,267 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:45:12,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:15,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:45:16,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:45:16,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:45:18,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:24,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:25,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:45:30,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:31,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:45:31,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:45:35,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:45:36,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:41,269 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:45:42,271 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:45,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:45:46,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:45:46,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:45:47,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:52,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:45:55,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:45:58,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:01,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:46:01,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:46:03,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:05,923 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:46:09,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:11,277 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:46:14,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:15,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:46:16,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:46:16,476 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:46:20,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:25,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:46:26,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:31,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:46:31,477 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:46:32,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:35,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:46:37,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:41,280 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:46:43,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:45,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:46:46,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:46:46,477 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:46:49,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:55,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:46:55,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:47:01,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:47:01,477 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:47:01,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:05,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:47:07,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:11,286 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:47:13,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:15,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:47:16,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:47:16,477 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:47:19,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:25,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:25,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:47:31,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:47:31,477 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:47:31,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:35,924 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:47:36,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:41,292 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:47:42,295 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:45,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:47:46,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:47:46,477 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:47:47,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:52,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:47:55,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:47:57,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:01,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:48:01,478 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:48:03,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:05,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:48:08,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:11,298 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:48:14,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:15,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:48:16,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:48:16,477 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:48:20,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:25,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:25,925 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:48:31,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:48:31,478 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:48:31,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:35,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:48:37,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:41,297 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:48:42,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:45,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:48:46,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:48:46,478 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:48:48,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:53,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:48:55,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:48:59,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:01,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:49:01,478 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:49:05,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:05,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:49:11,308 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:49:11,310 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:12,323 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:49:12,327 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:49:12,327 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:49:12,330 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:49:13,232 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:49:15,237 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:49:15,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:49:16,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:49:16,479 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:49:16,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:22,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:25,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:49:28,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:31,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:49:31,478 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:49:34,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:35,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:49:40,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:41,311 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:49:45,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:45,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:49:46,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:49:46,479 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:49:51,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:49:55,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:49:57,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:01,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:50:01,479 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:50:03,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:05,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:50:08,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:11,317 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:50:14,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:15,926 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:50:16,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:50:16,479 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:50:20,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:25,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:50:26,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:31,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:50:31,479 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:50:31,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:35,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:50:37,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:41,323 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:50:42,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:45,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:50:46,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:50:46,479 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:50:48,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:53,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:50:55,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:50:59,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:01,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:51:01,480 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:51:05,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:05,927 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:51:10,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:11,331 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:51:15,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:51:16,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:51:16,481 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:51:16,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:21,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:25,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:51:27,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:31,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:51:31,480 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:51:33,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:35,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:51:39,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:41,335 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:51:44,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:45,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:51:46,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:51:46,480 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:51:50,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:55,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:51:55,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:52:01,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:52:01,481 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:52:01,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:05,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:52:06,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:11,339 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:52:12,342 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:15,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:52:16,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:52:16,480 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:52:17,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:23,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:25,928 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:52:29,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:31,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:52:31,481 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:52:35,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:35,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:52:41,345 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:52:41,350 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:45,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:52:46,479 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:52:46,481 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:52:46,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:51,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:52:55,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:52:56,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:01,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:53:01,481 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:53:02,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:05,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:53:07,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:11,351 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:53:13,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:15,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:53:16,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:53:16,481 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:53:18,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:24,665 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:25,929 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:53:30,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:32,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:53:32,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:53:35,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:53:36,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:41,355 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:53:42,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:45,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:53:47,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:53:47,630 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:53:47,883 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:53,666 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:53:55,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:53:59,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:02,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:54:02,510 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:54:04,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:05,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:54:10,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:11,361 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:54:15,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:54:16,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:17,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:54:17,511 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:54:21,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:25,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:54:27,667 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:32,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:54:32,511 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:54:32,733 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:35,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:54:38,668 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:41,367 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:54:44,361 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 01:54:44,364 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 01:54:44,365 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 01:54:44,365 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:44,368 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 01:54:44,468 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 01:54:45,472 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 01:54:45,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:54:47,509 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:54:47,510 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:54:49,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:55,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:54:55,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:55:01,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:02,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:55:02,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:55:05,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:55:06,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:11,373 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:55:12,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:15,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:55:17,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:55:17,511 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:55:17,728 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:23,669 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:25,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:55:28,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:32,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:55:32,511 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:55:33,734 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:35,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:55:39,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:41,379 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:55:45,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:45,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:55:47,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:55:47,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:55:50,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:55:55,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:55:56,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:02,510 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:56:02,511 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:56:02,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:05,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:56:08,670 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:11,386 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:56:14,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:15,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:56:17,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:56:17,511 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:56:19,751 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:25,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:25,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:56:31,671 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:32,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:56:32,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:56:35,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:56:36,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:41,390 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:56:42,393 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:45,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:56:47,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:56:47,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:56:47,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:53,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:56:55,932 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:56:59,672 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:02,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:57:02,511 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:57:04,753 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:05,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:57:10,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:11,396 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:57:15,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:57:16,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:17,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:57:17,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:57:21,747 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:25,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:57:27,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:32,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:57:32,513 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:57:32,711 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:35,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:57:38,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:41,402 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:57:44,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:45,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:57:47,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:57:47,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:57:49,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:55,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:57:55,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:58:00,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:02,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:58:02,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:58:05,740 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:05,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:58:11,408 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:58:11,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:15,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:58:16,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:17,511 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:58:17,512 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:58:21,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:25,933 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:58:27,674 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:32,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:58:32,513 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:58:33,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:35,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:58:39,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:41,413 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:58:44,675 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:45,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:58:47,512 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:58:47,513 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:58:49,747 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:55,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:58:55,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:59:01,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:02,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:59:02,514 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:59:05,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:59:07,373 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:11,419 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:59:12,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:15,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:59:17,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:59:17,514 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:59:17,749 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:23,676 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:25,934 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:59:28,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:32,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:59:32,513 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:59:33,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:35,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:59:39,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:41,422 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 01:59:45,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:45,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:59:47,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 01:59:47,515 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 01:59:50,721 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 01:59:55,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 01:59:56,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:02,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:00:02,514 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:00:02,770 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:05,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:00:08,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:11,427 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:00:14,677 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:15,384 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:00:15,387 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:00:15,388 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:00:15,391 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:00:15,726 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:00:15,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:00:17,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:00:17,513 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:00:17,733 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:00:19,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:25,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:25,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:00:31,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:32,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:00:32,514 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:00:35,935 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:00:37,678 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:41,428 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:00:42,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:45,936 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:00:47,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:00:47,514 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:00:47,717 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:53,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:00:55,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:00:59,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:02,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:01:02,515 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:01:04,722 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:05,936 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:01:10,679 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:11,435 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:01:15,936 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:01:16,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:17,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:01:17,515 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:01:21,759 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:25,936 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:01:27,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:32,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:01:32,515 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:01:32,745 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:35,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:01:38,680 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:41,440 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:01:44,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:45,936 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:01:47,513 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:01:47,514 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:01:49,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:55,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:01:55,936 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:02:01,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:02,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:02:02,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:02:05,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:02:06,745 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:11,446 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:02:12,448 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:15,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:02:17,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:02:17,515 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:02:17,726 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:23,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:25,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:02:29,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:32,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:02:32,515 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:02:34,759 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:35,937 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:02:40,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:41,451 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:02:45,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:02:46,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:47,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:02:47,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:02:51,758 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:02:55,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:02:57,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:02,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:03:02,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:03:03,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:05,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:03:08,682 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:11,457 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:03:14,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:15,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:03:17,514 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:03:17,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:03:19,743 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:25,683 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:25,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:03:30,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:32,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:03:32,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:03:35,771 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:35,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:03:41,463 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:03:41,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:45,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:03:46,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:47,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:03:47,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:03:52,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:03:55,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:03:58,684 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:02,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:04:02,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:04:03,729 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:05,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:04:09,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:11,468 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:04:14,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:15,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:04:17,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:04:17,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:04:19,719 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:25,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:25,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:04:31,685 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:32,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:04:32,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:04:35,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:04:36,716 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:41,474 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:04:42,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:45,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:04:47,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:04:47,517 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:04:47,753 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:53,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:04:55,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:04:59,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:02,515 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:05:02,516 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:05:04,727 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:05,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:05:10,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:11,479 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:05:15,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:15,939 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:05:17,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:05:17,517 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:05:21,686 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:25,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:05:26,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:31,687 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:32,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:05:32,517 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:05:35,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:05:36,752 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:41,483 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:05:42,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:45,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:05:47,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:05:47,517 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:05:47,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:05:47,715 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:05:47,715 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:47,716 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:05:47,720 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:05:47,993 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:05:50,007 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:05:53,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:05:55,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:05:59,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:02,516 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:06:02,517 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:06:04,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:05,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:06:10,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:11,491 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:06:15,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:15,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:06:17,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:06:17,517 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:06:20,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:25,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:06:26,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:32,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:06:32,518 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:06:32,767 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:35,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:06:38,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:41,495 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:06:44,688 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:45,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:06:47,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:06:47,518 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:06:49,751 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:55,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:06:55,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:07:01,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:02,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:07:02,518 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:07:05,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:07:06,757 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:11,500 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:07:12,508 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:15,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:07:17,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:07:17,518 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:07:17,763 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:23,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:25,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:07:29,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:32,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:07:32,518 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:07:34,690 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:35,941 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:07:39,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:41,506 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:07:45,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:45,942 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:07:47,517 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:07:47,518 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:07:50,724 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:07:55,942 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:07:56,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:02,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:08:02,518 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:08:02,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:05,942 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:08:08,691 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:11,511 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:08:13,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:15,942 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:08:17,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:08:17,519 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:08:18,724 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:24,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:25,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:08:29,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:32,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:08:32,519 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:08:34,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:35,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:08:40,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:41,517 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:08:45,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:08:46,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:47,518 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:08:47,519 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:08:52,692 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:08:55,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:08:58,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:02,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:09:02,521 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:09:03,724 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:05,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:09:09,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:11,523 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:09:15,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:15,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:09:17,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:09:17,521 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:09:20,736 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:25,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:09:26,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:32,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:09:32,521 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:09:32,771 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:35,943 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:09:38,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:41,529 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:09:44,694 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:45,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:09:47,520 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:09:47,521 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:09:49,760 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:55,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:09:55,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:10:00,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:02,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:10:02,522 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:10:05,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:10:06,695 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:11,535 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:10:12,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:15,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:10:17,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:10:17,522 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:10:17,741 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:23,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:25,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:10:29,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:32,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:10:32,522 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:10:35,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:35,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:10:41,537 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:10:41,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:45,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:10:46,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:47,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:10:47,522 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:10:51,718 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:10:55,944 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:10:57,696 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:02,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:11:02,522 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:11:05,022 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:05,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:11:10,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:11,546 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:11:15,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:15,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:11:17,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:11:17,523 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:11:19,840 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:11:19,843 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:11:19,844 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:11:19,848 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:11:20,284 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:11:20,849 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:22,295 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:11:25,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:11:26,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:32,521 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:11:32,522 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:11:33,582 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:35,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:11:38,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:41,551 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:11:43,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:45,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:11:47,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:11:47,523 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:11:48,730 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:54,698 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:11:55,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:12:00,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:02,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:12:02,523 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:12:05,755 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:05,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:12:11,556 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:12:11,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:15,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:12:16,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:17,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:12:17,522 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:12:21,773 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:25,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:12:27,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:32,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:12:32,524 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:12:33,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:35,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:12:39,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:41,562 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:12:44,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:45,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:12:47,522 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:12:47,524 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:12:49,752 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:55,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:12:55,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:13:01,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:02,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:13:02,523 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:13:05,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:13:06,911 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:11,566 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:13:12,569 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:15,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:13:17,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:13:17,523 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:13:17,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:23,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:25,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:13:29,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:32,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:13:32,524 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:13:35,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:35,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:13:40,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:41,571 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:13:45,947 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:13:46,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:47,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:13:47,524 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:13:51,747 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:13:55,946 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:13:57,701 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:02,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:14:02,526 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:14:04,956 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:05,947 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:14:10,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:11,575 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:14:15,947 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:14:16,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:17,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:14:17,524 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:14:21,755 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:25,947 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:14:27,702 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:32,523 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:14:32,524 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:14:32,735 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:35,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:14:38,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:41,581 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:14:43,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:45,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:14:47,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:14:47,525 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:14:48,747 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:54,703 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:14:55,947 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:14:59,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:02,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:15:02,525 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:15:04,719 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:05,947 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:15:10,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:11,586 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:15:15,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:15:16,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:17,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:15:17,524 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:15:22,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:25,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:15:28,704 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:32,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:15:32,525 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:15:33,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:35,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:15:39,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:41,595 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:15:45,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:45,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:15:47,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:15:47,525 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:15:50,772 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:15:55,948 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:15:56,705 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:02,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:16:02,525 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:16:04,963 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:05,949 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:16:10,706 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:11,599 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:16:15,949 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:16:16,706 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:17,524 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:16:17,525 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:16:22,706 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:25,949 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:16:28,706 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:32,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:16:32,526 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:16:33,753 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:35,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:16:39,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:41,604 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:16:45,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:45,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:16:47,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:16:47,526 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:16:50,930 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:16:50,934 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:16:50,935 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:16:50,938 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:16:50,951 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:16:51,464 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:16:52,466 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:16:55,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:16:56,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:02,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:17:02,526 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:17:02,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:05,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:17:08,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:11,608 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:17:14,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:15,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:17:17,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:17:17,526 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:17:19,718 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:25,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:25,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:17:30,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:32,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:17:32,526 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:17:35,720 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:35,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:17:41,615 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:17:41,618 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:45,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:17:46,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:47,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:17:47,526 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:17:51,752 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:17:55,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:17:57,708 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:02,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:18:02,527 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:18:02,749 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:05,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:18:08,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:11,619 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:18:14,709 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:15,950 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:18:17,525 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:18:17,527 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:18:19,782 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:25,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:25,951 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:18:31,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:32,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:18:32,527 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:18:35,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:18:36,733 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:41,623 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:18:42,626 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:45,951 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:18:47,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:18:47,529 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:18:47,753 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:53,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:18:55,951 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:18:58,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:02,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:19:02,527 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:19:03,766 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:05,951 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:19:09,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:11,629 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:19:15,711 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:15,951 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:19:17,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:19:17,527 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:19:20,784 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:25,951 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:19:26,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:32,526 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:19:32,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:19:32,745 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:35,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:19:38,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:41,635 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:19:44,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:45,956 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:19:47,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:19:47,527 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:19:49,747 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:55,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:19:55,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:20:00,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:02,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:20:02,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:20:05,765 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:05,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:20:11,639 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:20:11,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:15,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:20:16,713 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:17,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:20:17,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:20:21,776 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:25,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:20:27,713 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:32,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:20:32,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:20:33,713 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:35,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:20:38,713 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:41,648 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:20:44,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:45,952 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:20:47,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:20:47,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:20:50,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:55,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:20:55,953 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:21:01,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:02,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:21:02,529 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:21:05,953 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:21:06,733 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:11,648 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:21:12,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:15,953 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:21:17,527 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:21:17,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:21:17,763 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:23,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:25,953 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:21:28,715 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:32,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:21:32,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:21:33,715 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:35,954 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:21:39,715 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:41,656 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:21:44,716 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:45,953 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:21:47,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:21:47,528 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:21:49,728 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:55,716 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:21:55,954 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:22:01,716 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:02,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:22:02,529 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:22:05,954 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:22:06,723 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:11,660 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:22:12,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:15,954 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:22:17,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:22:17,530 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:22:17,779 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:23,237 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:22:23,241 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:22:23,242 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:22:23,242 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:23,246 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:22:23,776 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:22:24,778 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:22:28,248 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:29,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:22:30,796 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:22:32,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:22:32,529 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:22:35,263 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:35,956 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:22:39,832 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:22:40,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:41,664 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:22:45,957 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:22:46,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:47,528 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:22:47,529 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:22:52,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:22:55,957 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:22:57,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:02,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:02,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:23:02,530 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:23:05,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:23:08,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:11,670 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:23:13,673 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:15,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:23:17,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:23:17,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:23:18,773 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:24,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:25,957 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:23:30,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:32,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:23:32,530 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:23:35,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:23:36,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:41,676 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:23:41,681 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:45,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:23:47,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:47,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:23:47,530 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:23:53,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:23:55,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:23:59,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:02,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:24:02,530 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:24:04,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:05,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:24:10,400 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:11,681 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:24:15,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:15,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:24:17,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:24:17,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:24:21,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:25,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:24:27,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:32,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:24:32,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:24:32,756 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:35,958 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:24:38,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:41,687 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:24:43,689 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:45,959 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:24:47,529 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:24:47,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:24:48,693 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:54,401 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:24:55,959 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:24:59,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:02,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:25:02,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:25:04,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:05,959 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:25:10,402 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:11,693 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:25:15,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:15,959 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:25:17,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:25:17,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:25:21,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:25,959 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:25:27,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:32,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:25:32,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:25:32,722 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:35,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:25:38,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:41,697 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:25:43,699 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:45,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:25:47,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:25:47,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:25:48,736 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:54,403 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:25:55,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:26:00,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:02,530 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:26:02,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:26:05,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:26:06,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:11,703 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:26:11,712 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:15,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:26:17,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:17,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:26:17,531 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:26:23,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:25,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:26:29,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:32,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:26:32,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:26:35,404 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:35,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:26:40,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:41,707 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:26:45,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:26:46,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:47,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:26:47,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:26:52,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:26:55,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:26:58,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:02,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:27:02,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:27:03,740 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:05,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:27:09,405 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:11,714 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:27:14,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:15,960 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:27:17,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:27:17,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:27:19,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:25,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:25,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:27:31,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:32,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:27:32,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:27:35,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:27:36,786 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:41,719 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:27:42,722 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:45,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:27:47,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:27:47,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:27:47,744 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:53,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:27:55,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:27:59,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:02,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:28:02,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:28:05,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:05,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:28:09,713 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:28:09,716 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:28:09,717 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:28:09,720 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:28:09,993 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:28:10,755 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:11,725 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:28:11,999 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:28:15,961 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:28:16,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:17,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:28:17,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:28:22,407 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:25,962 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:28:27,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:32,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:28:32,533 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:28:32,744 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:35,962 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:28:38,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:41,730 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:28:43,733 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:45,962 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:28:47,531 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:28:47,532 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:28:48,782 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:54,408 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:28:55,962 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:28:59,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:02,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:29:02,533 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:29:04,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:06,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:29:10,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:11,735 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:29:16,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:29:16,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:17,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:29:17,533 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:29:22,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:26,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:29:27,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:32,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:32,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:29:32,534 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:29:36,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:29:37,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:41,739 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:29:42,742 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:46,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:29:47,532 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:29:47,533 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:29:47,776 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:53,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:29:56,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:29:59,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:02,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:30:02,534 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:30:05,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:06,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:30:11,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:11,747 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:30:16,375 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:30:16,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:17,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:30:17,534 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:30:22,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:26,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:30:28,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:32,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:30:32,534 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:30:33,772 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:36,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:30:39,411 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:41,751 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:30:45,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:46,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:30:47,533 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:30:47,534 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:30:51,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:30:56,376 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:30:56,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:02,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:02,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:31:02,535 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:31:06,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:31:08,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:11,753 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:31:13,756 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:16,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:31:17,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:31:17,535 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:31:19,412 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:25,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:26,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:31:31,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:32,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:31:32,535 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:31:36,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:31:37,413 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:41,761 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:31:42,764 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:46,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:31:47,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:31:47,535 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:31:47,793 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:53,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:31:56,377 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:31:59,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:02,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:32:02,535 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:32:05,414 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:06,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:32:10,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:11,768 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:32:16,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:32:16,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:17,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:32:17,535 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:32:22,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:26,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:32:28,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:32,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:32:32,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:32:33,727 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:36,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:32:39,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:41,771 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:32:44,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:46,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:32:47,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:32:47,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:32:50,415 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:32:56,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:32:56,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:02,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:02,534 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:33:02,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:33:06,378 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:33:07,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:11,776 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:33:12,778 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:16,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:33:17,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:33:17,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:33:18,777 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:24,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:26,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:33:30,416 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:32,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:33:32,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:33:36,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:33:36,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:41,782 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:33:41,787 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:42,096 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:33:42,100 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:33:42,106 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:33:42,110 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:33:42,293 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:33:44,298 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:33:46,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:33:47,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:47,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:33:47,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:33:53,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:33:56,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:33:59,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:02,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:34:02,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:34:04,417 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:06,379 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:34:09,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:11,787 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:34:15,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:16,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:34:17,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:34:17,537 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:34:21,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:26,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:34:27,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:32,535 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:34:32,536 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:34:32,789 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:36,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:34:38,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:41,791 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:34:43,794 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:46,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:34:47,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:34:47,537 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:34:49,418 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:55,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:34:56,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:35:01,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:02,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:35:02,537 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:35:06,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:35:07,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:11,799 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:35:12,801 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:16,380 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:35:17,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:35:17,537 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:35:18,710 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:24,419 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:26,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:35:30,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:32,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:35:32,538 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:35:36,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:35:36,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:41,801 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:35:41,813 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:46,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:35:47,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:47,536 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:35:47,537 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:35:53,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:35:56,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:35:59,420 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:02,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:36:02,537 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:36:04,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:06,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:36:10,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:11,806 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:36:16,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:36:16,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:17,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:36:17,538 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:36:22,421 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:26,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:36:27,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:32,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:36:32,538 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:36:32,788 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:36,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:36:38,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:41,816 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:36:43,819 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:46,381 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:36:47,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:36:47,538 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:36:49,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:55,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:36:56,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:37:01,422 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:02,537 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:37:02,538 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:37:06,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:37:07,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:11,817 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:37:12,820 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:16,382 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:37:17,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:37:17,539 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:37:18,714 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:24,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:26,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:37:30,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:32,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:37:32,538 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:37:36,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:37:36,423 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:41,824 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:37:41,829 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:46,383 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:37:47,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:47,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:37:47,538 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:37:53,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:37:56,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:37:59,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:02,538 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:38:02,539 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:38:05,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:06,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:38:11,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:11,828 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:38:16,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:38:17,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:17,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:38:17,544 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:38:23,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:26,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:38:29,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:32,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:38:32,544 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:38:35,424 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:36,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:38:40,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:41,835 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:38:46,425 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:46,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:38:47,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:38:47,544 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:38:51,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:38:56,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:38:57,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:02,543 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:39:02,544 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:39:04,794 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:06,627 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:39:10,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:11,838 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:39:14,332 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:39:14,336 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:39:14,336 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:39:14,340 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:39:14,513 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:39:16,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:16,520 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:39:16,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:39:17,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:39:17,544 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:39:22,426 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:26,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:39:27,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:32,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:39:32,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:39:32,792 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:36,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:39:38,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:41,848 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:39:43,851 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:46,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:39:47,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:39:47,545 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:39:49,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:55,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:39:56,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:40:01,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:02,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:40:02,545 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:40:06,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:40:07,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:11,853 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:40:12,856 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:16,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:40:17,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:40:17,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:40:18,754 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:24,427 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:26,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:40:30,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:32,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:40:32,545 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:40:36,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:36,628 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:40:41,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:41,855 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:40:46,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:40:47,428 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:47,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:40:47,545 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:40:52,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:40:56,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:40:58,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:02,544 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:41:02,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:41:03,718 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:06,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:41:09,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:11,863 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:41:15,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:16,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:41:17,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:41:17,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:41:21,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:26,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:41:27,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:32,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:41:32,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:41:32,753 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:36,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:41:38,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:41,867 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:41:43,870 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:46,629 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:41:47,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:41:47,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:41:49,429 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:55,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:41:56,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:42:01,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:02,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:42:02,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:42:06,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:42:07,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:11,873 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:42:12,876 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:16,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:42:17,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:42:17,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:42:18,752 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:24,430 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:26,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:42:29,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:32,545 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:42:32,546 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:42:35,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:36,630 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:42:41,431 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:41,879 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:42:46,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:46,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:42:47,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:42:47,547 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:42:52,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:42:56,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:42:58,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:02,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:43:02,547 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:43:03,697 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:06,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:43:09,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:11,884 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:43:14,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:16,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:43:17,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:43:17,547 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:43:20,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:26,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:26,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:43:32,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:32,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:43:32,548 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:43:36,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:43:38,432 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:41,889 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:43:43,891 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:46,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:43:47,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:43:47,547 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:43:49,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:55,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:43:56,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:44:01,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:02,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:44:02,548 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:44:06,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:44:07,433 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:11,895 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:44:12,898 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:16,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:44:17,546 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:44:17,547 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:44:18,783 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:24,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:26,631 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:44:30,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:32,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:44:32,548 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:44:35,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:36,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:44:41,434 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:41,906 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:44:45,309 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:44:45,312 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:44:45,313 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:44:45,316 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:44:45,705 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:44:46,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:44:46,743 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:44:47,352 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:47,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:44:47,548 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:44:52,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:44:56,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:44:58,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:02,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:45:02,548 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:45:03,744 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:06,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:45:09,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:11,903 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:45:15,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:16,632 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:45:17,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:45:17,548 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:45:21,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:26,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:45:27,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:32,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:45:32,548 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:45:32,758 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:36,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:45:38,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:41,910 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:45:43,913 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:46,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:45:47,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:45:47,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:45:49,435 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:54,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:45:56,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:46:00,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:02,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:46:02,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:46:05,547 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:06,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:46:11,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:11,916 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:46:16,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:46:17,436 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:17,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:46:17,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:46:22,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:26,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:46:28,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:32,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:46:32,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:46:33,778 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:36,633 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:46:39,437 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:41,922 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:46:45,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:46,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:46:47,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:46:47,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:46:51,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:46:56,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:46:57,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:02,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:47:02,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:47:02,761 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:06,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:47:08,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:11,928 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:47:13,931 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:16,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:47:17,548 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:47:17,549 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:47:19,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:25,438 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:26,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:47:31,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:32,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:47:32,550 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:47:36,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:36,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:47:41,934 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:47:41,940 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:46,634 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:47:47,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:47,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:47:47,550 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:47:53,439 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:47:56,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:47:58,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:02,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:48:02,550 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:48:03,757 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:06,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:48:09,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:11,940 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:48:15,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:16,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:48:17,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:48:17,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:48:21,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:26,635 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:48:27,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:32,549 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:48:32,550 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:48:32,719 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:36,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:48:38,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:41,942 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:48:43,945 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:46,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:48:47,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:48:47,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:48:49,440 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:54,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:48:56,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:49:00,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:02,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:49:02,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:49:06,115 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:06,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:49:11,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:11,949 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:49:16,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:49:17,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:17,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:49:17,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:49:23,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:26,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:49:29,441 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:32,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:49:32,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:49:34,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:36,636 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:49:40,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:41,956 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:49:45,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:46,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:49:47,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:49:47,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:49:51,442 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:56,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:49:56,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:50:02,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:02,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:50:02,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:50:06,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:50:08,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:11,959 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:50:13,962 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:16,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:50:17,357 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:50:17,360 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:50:17,361 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:50:17,365 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:50:17,550 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:50:17,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:50:17,965 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:50:18,968 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:50:19,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:25,443 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:26,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:50:31,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:32,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:50:32,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:50:36,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:36,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:50:41,965 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:50:41,970 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:46,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:50:47,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:47,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:50:47,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:50:53,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:50:56,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:50:59,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:02,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:51:02,552 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:51:05,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:06,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:51:11,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:11,970 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:51:16,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:16,637 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:51:17,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:51:17,551 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:51:22,445 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:26,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:51:28,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:32,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:51:32,552 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:51:33,707 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:36,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:51:39,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:41,972 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:51:44,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:46,638 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:51:47,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:51:47,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:51:50,446 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:56,447 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:51:56,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:52:01,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:02,551 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:52:02,552 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:52:06,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:06,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:52:11,981 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:52:11,983 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:16,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:52:17,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:17,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:52:17,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:52:23,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:26,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:52:29,451 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:32,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:52:32,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:52:35,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:36,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:52:41,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:41,980 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:52:46,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:52:47,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:47,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:52:47,554 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:52:53,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:52:56,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:52:59,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:02,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:53:02,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:53:05,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:06,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:53:10,453 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:11,987 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:53:15,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:16,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:53:17,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:53:17,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:53:21,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:26,639 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:53:27,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:32,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:53:32,554 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:53:32,752 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:36,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:53:38,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:41,991 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:53:43,992 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:46,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:53:47,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:53:47,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:53:49,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:55,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:53:56,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:54:01,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:02,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:54:02,554 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:54:06,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:54:07,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:11,997 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:54:13,000 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:16,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:54:17,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:54:17,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:54:18,757 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:24,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:26,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:54:30,455 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:32,552 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:54:32,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:54:35,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:36,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:54:41,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:42,007 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:54:46,640 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:54:47,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:47,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:54:47,553 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:54:53,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:54:56,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:54:58,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:02,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:55:02,554 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:55:03,757 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:06,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:55:09,456 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:12,008 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:55:14,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:16,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:55:17,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:55:17,554 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:55:20,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:26,457 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:26,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:55:31,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:32,553 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:55:32,554 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:55:36,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:55:37,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:42,012 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:55:43,015 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:46,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:55:47,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:55:47,555 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:55:48,718 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:49,192 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 02:55:49,195 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 02:55:49,196 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 02:55:49,206 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 02:55:50,173 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 02:55:51,176 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 02:55:54,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:55:56,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:55:59,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:02,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:56:02,555 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:56:05,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:06,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:56:10,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:12,020 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:56:16,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:16,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:56:17,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:56:17,555 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:56:22,458 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:26,641 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:56:27,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:32,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:32,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:56:32,555 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:56:36,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:56:38,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:42,025 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:56:44,028 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:46,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:56:47,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:56:47,555 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:56:49,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:54,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:56:56,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:56:59,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:02,554 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:57:02,555 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:57:05,460 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:06,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:57:10,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:12,031 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:57:16,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:16,642 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:57:17,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:57:17,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:57:22,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:26,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:57:27,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:32,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:57:32,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:57:32,807 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:36,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:57:38,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:42,036 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:57:44,038 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:46,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:57:47,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:57:47,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:57:49,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:55,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:57:56,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:58:01,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:02,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:58:02,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:58:06,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:58:07,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:12,038 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:58:13,040 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:16,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:58:17,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:58:17,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:58:18,779 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:24,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:26,643 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:58:30,462 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:32,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:58:32,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:58:36,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:36,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:58:41,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:42,042 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:58:46,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:58:47,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:47,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:58:47,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:58:53,463 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:58:56,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:58:58,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:02,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:59:02,558 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:59:03,700 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:06,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:59:09,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:12,049 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:59:14,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:16,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:59:17,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:59:17,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:59:20,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:25,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:26,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:59:31,464 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:32,555 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:59:32,556 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:59:36,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:36,644 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:59:42,056 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 02:59:42,061 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:46,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:59:47,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:47,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 02:59:47,557 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 02:59:53,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 02:59:56,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 02:59:59,465 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:02,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:00:02,557 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:00:05,114 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:06,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:00:10,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:12,059 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:00:16,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:16,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:00:17,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:00:17,557 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:00:22,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:26,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:00:28,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:32,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:00:32,557 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:00:33,738 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:36,646 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:00:39,466 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:42,064 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:00:45,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:46,645 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:00:47,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:00:47,557 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:00:51,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:00:56,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:00:57,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:02,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:02,556 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:01:02,557 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:01:06,649 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:01:07,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:12,069 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:01:13,071 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:16,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:01:17,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:01:17,557 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:01:18,775 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:20,351 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 03:01:20,354 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 03:01:20,355 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 03:01:20,358 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 03:01:20,465 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 03:01:21,468 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 03:01:24,360 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:26,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:01:29,467 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:32,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:01:32,561 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:01:34,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:36,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:01:39,468 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:42,073 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:01:45,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:46,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:01:47,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:01:47,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:01:51,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:56,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:01:56,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:02:02,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:02,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:02:02,558 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:02:06,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:02:08,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:12,081 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:02:14,083 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:16,650 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:02:17,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:02:17,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:02:19,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:25,469 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:26,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:02:30,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:32,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:02:32,558 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:02:36,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:36,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:02:42,085 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:02:42,089 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:46,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:02:47,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:47,557 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:02:47,560 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:02:53,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:02:56,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:02:59,470 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:02,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:03:02,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:03:04,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:06,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:03:10,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:12,091 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:03:15,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:16,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:03:17,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:03:17,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:03:21,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:26,651 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:03:27,471 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:32,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:03:32,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:03:32,803 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:36,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:03:38,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:42,097 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:03:44,099 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:46,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:03:47,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:03:47,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:03:49,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:55,472 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:03:56,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:04:01,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:02,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:04:02,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:04:06,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:04:07,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:12,106 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:04:13,109 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:16,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:04:17,558 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:04:17,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:04:18,711 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:24,473 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:26,652 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:04:29,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:32,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:04:32,559 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:04:35,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:36,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:04:41,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:42,100 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:04:46,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:04:47,474 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:47,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:04:47,560 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:04:53,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:04:56,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:04:58,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:02,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:05:02,560 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:05:03,764 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:06,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:05:09,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:12,103 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:05:14,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:16,653 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:05:17,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:05:17,560 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:05:20,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:26,475 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:26,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:05:32,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:32,559 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:05:32,560 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:05:36,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:05:38,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:42,105 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:05:44,108 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:46,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:05:47,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:05:47,561 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:05:49,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:54,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:05:56,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:06:00,476 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:02,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:06:02,561 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:06:06,137 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:06,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:06:11,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:12,113 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:06:16,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:16,654 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:06:17,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:06:17,561 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:06:22,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:26,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:06:28,477 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:32,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:06:32,561 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:06:33,724 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:36,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:06:39,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:42,113 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:06:45,478 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:46,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:06:47,560 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:06:47,561 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:06:50,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:52,744 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 03:06:52,748 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 03:06:52,749 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 03:06:52,752 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 03:06:52,831 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 03:06:53,833 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 03:06:56,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:06:56,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:07:01,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:02,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:07:02,561 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:07:06,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:07:07,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:12,117 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:07:13,120 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:16,655 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:07:17,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:07:17,567 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:07:18,813 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:24,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:26,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:07:29,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:32,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:07:32,562 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:07:35,480 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:36,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:07:40,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:42,123 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:07:46,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:46,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:07:47,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:07:47,562 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:07:52,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:07:56,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:07:58,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:02,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:08:02,562 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:08:03,773 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:06,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:08:09,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:12,128 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:08:15,481 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:16,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:08:17,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:08:17,562 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:08:21,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:26,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:08:27,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:32,561 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:08:32,562 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:08:32,793 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:36,656 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:08:38,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:42,134 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:08:44,136 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:46,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:08:47,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:08:47,562 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:08:49,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:54,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:08:56,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:09:00,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:02,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:09:02,563 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:09:06,482 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:06,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:09:12,142 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:09:12,145 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:16,657 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:09:17,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:17,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:09:17,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:09:23,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:26,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:09:28,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:32,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:09:32,565 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:09:33,822 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:36,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:09:39,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:42,143 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:09:45,483 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:46,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:09:47,562 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:09:47,563 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:09:50,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:56,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:09:56,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:10:02,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:02,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:10:02,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:10:06,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:10:08,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:12,149 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:10:14,152 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:16,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:10:17,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:10:17,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:10:19,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:25,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:26,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:10:30,484 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:32,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:10:32,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:10:35,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:36,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:10:41,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:42,155 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:10:46,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:10:47,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:47,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:10:47,563 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:10:53,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:10:56,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:10:59,485 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:02,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:11:02,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:11:04,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:06,658 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:11:10,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:12,160 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:11:16,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:16,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:11:17,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:11:17,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:11:22,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:26,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:11:28,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:32,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:11:32,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:11:33,765 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:36,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:11:39,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:42,165 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:11:45,486 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:46,659 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:11:47,563 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:11:47,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:11:51,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:11:56,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:11:57,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:02,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:02,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:12:02,564 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:12:06,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:12:08,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:12,172 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:12:14,174 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:16,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:12:17,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:12:17,565 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:12:19,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:23,733 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: partial_history +2024-09-03 03:12:23,736 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 03:12:23,737 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 03:12:23,741 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 03:12:24,196 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 03:12:25,487 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:26,209 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 03:12:26,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:12:30,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:32,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:12:32,565 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:12:36,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:36,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:12:41,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:42,177 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:12:46,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:12:47,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:47,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:12:47,565 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:12:53,488 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:12:56,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:12:58,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:02,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:13:02,565 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:13:03,717 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:06,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:13:09,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:12,182 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:13:15,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:16,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:13:17,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:13:17,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:13:21,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:26,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:26,660 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:13:32,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:32,564 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:13:32,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:13:36,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:13:38,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:42,187 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:13:44,190 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:46,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:13:47,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:13:47,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:13:49,489 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:55,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:13:56,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:14:01,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:02,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:14:02,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:14:06,661 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:14:07,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:12,194 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:14:13,197 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:16,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:14:17,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:14:17,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:14:18,780 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:24,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:26,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:14:30,490 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:32,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:14:32,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:14:36,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:36,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:14:42,197 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:14:42,202 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:46,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:14:47,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:47,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:14:47,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:14:53,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:14:56,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:14:59,491 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:02,565 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:15:02,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:15:04,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:06,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:15:10,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:12,201 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:15:16,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:16,662 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:15:17,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:15:17,566 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:15:22,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:26,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:15:28,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:32,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:15:32,567 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:15:33,749 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:36,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:15:39,492 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:42,208 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:15:45,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:46,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:15:47,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:15:47,567 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:15:51,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:15:56,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:15:57,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:02,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:16:02,568 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:16:04,901 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:06,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:16:10,493 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:12,214 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:16:16,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:16,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:16:17,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:16:17,567 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:16:22,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:26,663 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:16:27,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:32,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:16:32,567 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:16:32,754 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:36,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:16:38,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:42,219 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:16:44,223 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:46,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:16:47,566 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:16:47,567 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:16:49,494 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:54,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:16:56,664 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:17:00,495 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:17:02,567 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: stop_status +2024-09-03 03:17:02,568 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: stop_status +2024-09-03 03:17:04,395 DEBUG SenderThread:6573 [sender.py:send():391] send: exit +2024-09-03 03:17:04,396 INFO SenderThread:6573 [sender.py:send_exit():598] handling exit code: 255 +2024-09-03 03:17:04,396 INFO SenderThread:6573 [sender.py:send_exit():600] handling runtime: 36838 +2024-09-03 03:17:04,399 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 03:17:04,405 INFO SenderThread:6573 [sender.py:send_exit():606] send defer +2024-09-03 03:17:04,406 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,407 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 0 +2024-09-03 03:17:04,407 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,408 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 0 +2024-09-03 03:17:04,408 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 1 +2024-09-03 03:17:04,409 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,409 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 1 +2024-09-03 03:17:04,409 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,409 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 1 +2024-09-03 03:17:04,410 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 2 +2024-09-03 03:17:04,410 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,411 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 2 +2024-09-03 03:17:04,411 INFO HandlerThread:6573 [system_monitor.py:finish():203] Stopping system monitor +2024-09-03 03:17:04,411 DEBUG SystemMonitor:6573 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-03 03:17:04,412 DEBUG SystemMonitor:6573 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-03 03:17:04,412 INFO HandlerThread:6573 [interfaces.py:finish():200] Joined cpu monitor +2024-09-03 03:17:04,416 INFO HandlerThread:6573 [interfaces.py:finish():200] Joined disk monitor +2024-09-03 03:17:04,438 INFO HandlerThread:6573 [interfaces.py:finish():200] Joined gpu monitor +2024-09-03 03:17:04,438 INFO HandlerThread:6573 [interfaces.py:finish():200] Joined memory monitor +2024-09-03 03:17:04,439 INFO HandlerThread:6573 [interfaces.py:finish():200] Joined network monitor +2024-09-03 03:17:04,443 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,443 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 2 +2024-09-03 03:17:04,443 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 3 +2024-09-03 03:17:04,444 DEBUG SenderThread:6573 [sender.py:send():391] send: stats +2024-09-03 03:17:04,444 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,446 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 3 +2024-09-03 03:17:04,447 DEBUG SenderThread:6573 [sender.py:send():391] send: history +2024-09-03 03:17:04,448 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: summary_record +2024-09-03 03:17:04,451 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 03:17:04,452 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,452 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 3 +2024-09-03 03:17:04,452 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 4 +2024-09-03 03:17:04,452 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,453 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 4 +2024-09-03 03:17:04,453 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,453 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 4 +2024-09-03 03:17:04,453 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 5 +2024-09-03 03:17:04,454 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,454 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 5 +2024-09-03 03:17:04,455 DEBUG SenderThread:6573 [sender.py:send():391] send: summary +2024-09-03 03:17:04,458 INFO SenderThread:6573 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-03 03:17:04,458 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,458 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 5 +2024-09-03 03:17:04,459 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 6 +2024-09-03 03:17:04,459 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,459 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 6 +2024-09-03 03:17:04,460 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,460 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 6 +2024-09-03 03:17:04,460 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 7 +2024-09-03 03:17:04,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: status_report +2024-09-03 03:17:04,461 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:04,461 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 7 +2024-09-03 03:17:04,461 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:04,462 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 7 +2024-09-03 03:17:05,328 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 03:17:05,395 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-03 03:17:07,336 INFO Thread-12 :6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 03:17:07,776 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 8 +2024-09-03 03:17:07,776 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: poll_exit +2024-09-03 03:17:07,776 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:07,777 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 8 +2024-09-03 03:17:07,778 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:07,778 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 8 +2024-09-03 03:17:07,779 INFO SenderThread:6573 [job_builder.py:build():443] Attempting to build job artifact +2024-09-03 03:17:07,782 INFO SenderThread:6573 [job_builder.py:_get_source_type():572] is repo sourced job +2024-09-03 03:17:07,782 WARNING SenderThread:6573 [job_builder.py:_log_if_verbose():274] No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job +2024-09-03 03:17:07,782 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 9 +2024-09-03 03:17:07,783 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:07,783 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 9 +2024-09-03 03:17:07,783 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:07,784 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 9 +2024-09-03 03:17:07,784 INFO SenderThread:6573 [dir_watcher.py:finish():358] shutting down directory watcher +2024-09-03 03:17:08,343 INFO SenderThread:6573 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 03:17:08,344 INFO SenderThread:6573 [dir_watcher.py:finish():388] scan: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files +2024-09-03 03:17:08,346 INFO SenderThread:6573 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json wandb-summary.json +2024-09-03 03:17:08,347 INFO SenderThread:6573 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log output.log +2024-09-03 03:17:08,353 INFO SenderThread:6573 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/requirements.txt requirements.txt +2024-09-03 03:17:08,359 INFO SenderThread:6573 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-metadata.json wandb-metadata.json +2024-09-03 03:17:08,360 INFO SenderThread:6573 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/config.yaml config.yaml +2024-09-03 03:17:08,366 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 10 +2024-09-03 03:17:08,367 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:08,371 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 10 +2024-09-03 03:17:08,372 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:08,373 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 10 +2024-09-03 03:17:08,373 INFO SenderThread:6573 [file_pusher.py:finish():169] shutting down file pusher +2024-09-03 03:17:08,398 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-03 03:17:08,398 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: poll_exit +2024-09-03 03:17:08,773 INFO wandb-upload_3:6573 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/config.yaml +2024-09-03 03:17:08,866 INFO wandb-upload_0:6573 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/wandb-summary.json +2024-09-03 03:17:08,894 INFO wandb-upload_2:6573 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/requirements.txt +2024-09-03 03:17:08,913 INFO wandb-upload_1:6573 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/files/output.log +2024-09-03 03:17:09,114 INFO Thread-11 (_thread_body):6573 [sender.py:transition_state():626] send defer: 11 +2024-09-03 03:17:09,115 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:09,116 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 11 +2024-09-03 03:17:09,117 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:09,117 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 11 +2024-09-03 03:17:09,118 INFO SenderThread:6573 [file_pusher.py:join():175] waiting for file pusher +2024-09-03 03:17:09,119 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 12 +2024-09-03 03:17:09,119 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:09,119 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 12 +2024-09-03 03:17:09,120 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:09,120 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 12 +2024-09-03 03:17:09,121 INFO SenderThread:6573 [file_stream.py:finish():601] file stream finish called +2024-09-03 03:17:09,399 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-03 03:17:09,599 INFO SenderThread:6573 [file_stream.py:finish():605] file stream finish is done +2024-09-03 03:17:09,599 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 13 +2024-09-03 03:17:09,605 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: poll_exit +2024-09-03 03:17:09,605 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:09,607 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 13 +2024-09-03 03:17:09,607 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:09,608 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 13 +2024-09-03 03:17:09,608 INFO SenderThread:6573 [sender.py:transition_state():626] send defer: 14 +2024-09-03 03:17:09,609 DEBUG SenderThread:6573 [sender.py:send():391] send: final +2024-09-03 03:17:09,609 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: defer +2024-09-03 03:17:09,610 DEBUG SenderThread:6573 [sender.py:send():391] send: footer +2024-09-03 03:17:09,610 INFO HandlerThread:6573 [handler.py:handle_request_defer():184] handle defer: 14 +2024-09-03 03:17:09,611 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: defer +2024-09-03 03:17:09,611 INFO SenderThread:6573 [sender.py:send_request_defer():622] handle sender defer: 14 +2024-09-03 03:17:09,612 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-03 03:17:09,613 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: poll_exit +2024-09-03 03:17:09,613 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: server_info +2024-09-03 03:17:09,614 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-03 03:17:09,615 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: server_info +2024-09-03 03:17:09,615 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: get_summary +2024-09-03 03:17:09,619 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: sampled_history +2024-09-03 03:17:09,625 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-03 03:17:09,756 DEBUG SenderThread:6573 [sender.py:send_request():418] send_request: poll_exit +2024-09-03 03:17:09,757 INFO MainThread:6573 [wandb_run.py:_footer_history_summary_info():4078] rendering history +2024-09-03 03:17:09,759 INFO MainThread:6573 [wandb_run.py:_footer_history_summary_info():4110] rendering summary +2024-09-03 03:17:09,760 INFO MainThread:6573 [wandb_run.py:_footer_sync_info():4037] logging synced files +2024-09-03 03:17:09,761 DEBUG HandlerThread:6573 [handler.py:handle_request():158] handle_request: shutdown +2024-09-03 03:17:09,762 INFO HandlerThread:6573 [handler.py:finish():884] shutting down handler +2024-09-03 03:17:10,615 INFO WriterThread:6573 [datastore.py:close():296] close: /workspace/nanoT5/logs/2024-09-02/17-03-02/wandb/run-20240902_170304-v43qltex/run-v43qltex.wandb +2024-09-03 03:17:10,757 INFO SenderThread:6573 [sender.py:finish():1631] shutting down sender +2024-09-03 03:17:10,757 INFO SenderThread:6573 [file_pusher.py:finish():169] shutting down file pusher +2024-09-03 03:17:10,758 INFO SenderThread:6573 [file_pusher.py:join():175] waiting for file pusher